diff --git a/dags/stat/evaluate/district_sell_property.py b/dags/stat/evaluate/district_sell_property.py index 46c13d6..f28477a 100644 --- a/dags/stat/evaluate/district_sell_property.py +++ b/dags/stat/evaluate/district_sell_property.py @@ -2,7 +2,7 @@ 统计近三个月、近一年样本量及小区属性 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils, commutils import pandas as pd import datetime import sys @@ -24,37 +24,8 @@ seaweed_df = pd.DataFrame(list(seaweed_conn.find({'cat': 'district', 'status': 0 .batch_size(10000))) seaweed_df = seaweed_df.rename(columns={'mapping_alias': 'alias'}) -def sell_stat(): - """ - 最近一年有样本小区占比 最近3月有样本小区占比 - """ - df = pd.read_excel(aim_file_name) - sell_seaweed_df = es206_client.scan_by_sql_to_data_frame( - "select city, region, district_name, p_date from latest_seawater where p_date > " + start_time + " and cat='sell' ") - # 小区别名升级为单独df - actions = list() - for index, item in seaweed_df.iterrows(): - if type(item['alias']) == list: - for name in item['alias']: - actions.append({'_id': item['_id'], 'city': item['city'], 'name': name, 'region': item['region']}) - alias_df = pd.DataFrame(actions) - - sell_seaweed_rename_df = sell_seaweed_df[['city', 'region', 'district_name']] - sell_seaweed_rename_df = sell_seaweed_rename_df.rename(columns={'district_name': 'name'}) - sell_seaweed_merge_df = pd.merge(sell_seaweed_rename_df, alias_df, how='left', on=['city', 'region', 'name']) - sell_seaweed_merge_df['count'] = sell_seaweed_merge_df[sell_seaweed_merge_df['_id'].notna()].groupby('_id')[ - '_id'].transform('size') - sell_seaweed_merge_df = sell_seaweed_merge_df[['city', 'region', '_id', 'count']] - sell_seaweed_merge_df = sell_seaweed_merge_df.drop_duplicates(subset=['city', 'region', '_id']) - sell_seaweed_ok_df = sell_seaweed_merge_df[sell_seaweed_merge_df['count'] > sell_num] - sell_seaweed_ok_df['city_seaweed_count'] = sell_seaweed_ok_df.groupby('city')['city'].transform('size') - city_seaweed_count_df = sell_seaweed_ok_df[['city', 'city_seaweed_count']].drop_duplicates( - subset=['city', 'city_seaweed_count']) - city_seaweed_count_df = city_seaweed_count_df.rename(columns={'city': '城市', 'city_seaweed_count': '最近3月有样本小区占比'}) - df = pd.merge(df, city_seaweed_count_df, how='left', on=['城市']) - df['最近3月有样本小区占比'] = round(df['最近3月有样本小区占比'] * 100 / df['小区量'], 2) - df.to_excel(aim_file_name, index=False) +def get_sell(df, daysBefore=365): cities = df[['城市']].drop_duplicates(keep='first')['城市'].values.tolist() city_one_year_count = list() for city in cities: @@ -79,8 +50,7 @@ def sell_stat(): names_str = "','".join(names) names_str = "'" + names_str + "'" try: - sql = "select count(*) from latest_seawater where city='" + city + "' and region='" + region + "' and cat='sell' and district_name in(" + names_str + ") and p_date > " + one_year_time - num = es206_client.search_by_sql(sql)['hits']['total']['value'] + num = commutils.get_sample(city, region, names_str, daysBefore=daysBefore) if num > sell_num: count += 1 total_num += num @@ -92,9 +62,25 @@ def sell_stat(): print(city + "已处理完, 近一年有样本小区数量" + str(count) + " 近一年的样本总量" + str(total_num)) result_df = pd.DataFrame(city_one_year_count) + return result_df + + +def sell_stat(): + """ + 最近一年有样本小区占比 最近3月有样本小区占比 + """ + df = pd.read_excel(aim_file_name) + result_df = get_sell(df, 365) result_df = result_df[['城市', '近一年有样本小区数量', '近一年的样本总量']] df = pd.merge(df, result_df, how='left', on=['城市']) - df['最近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2) + df['近一年有样本小区占比'] = round(df['近一年有样本小区数量'] * 100 / df['小区量'], 2) + + result_df = get_sell(df, 93) + result_df = result_df.rename(columns={'近一年有样本小区数量': '近三月有样本小区数量'}) + result_df = result_df[['城市', '近三月有样本小区数量']] + df = pd.merge(df, result_df, how='left', on=['城市']) + df['近三月有样本小区占比'] = round(df['近三月有样本小区数量'] * 100 / df['小区量'], 2) + df.to_excel(aim_file_name, index=False)