diff --git a/dags/business/business/community/data/全国各区户均面积已去极值.xlsx b/dags/business/business/community/data/全国各区户均面积已去极值.xlsx new file mode 100644 index 0000000..8e71f9e Binary files /dev/null and b/dags/business/business/community/data/全国各区户均面积已去极值.xlsx differ diff --git a/dags/business/business/community/total_price_complete.py b/dags/business/business/community/total_price_complete.py index 1afb0f7..3a94f63 100644 --- a/dags/business/business/community/total_price_complete.py +++ b/dags/business/business/community/total_price_complete.py @@ -22,6 +22,8 @@ def build_doc(item): def save(community_factor_df): + community_factor_df = community_factor_df[community_factor_df['total_price'].notna()] + community_factor_df['total_price'] = community_factor_df['total_price'].astype(int) actions = list() count = 0 for index, item in community_factor_df.iterrows(): @@ -46,37 +48,67 @@ def save(community_factor_df): actions.clear() -def sync(): - log.info(f"开始补全户均总价信息至206 {index_name}") - +def complete_by_area(community_df): factor_df = pd.read_csv(path + '/小区户均总价面积因子已去极值.csv', sep='@', encoding='UTF-8') - community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null") - community_factor_df = pd.merge(community_df, factor_df, how='left', on=['_id', 'city', 'region', 'name']) community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] + return community_factor_df + +def complete_by_build_year_area(community_factor_df): build_year_area_df = pd.read_excel(path + '/全国各区各建筑年代户均面积已去极值.xlsx') build_year_area_dict = dict() for index, build_year_area in build_year_area_df.iterrows(): - build_year_area_dict[build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor'] - for index, item in community_factor_df[(community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows(): + build_year_area_dict[ + build_year_area['city'] + '|' + build_year_area['region'] + "|" + build_year_area['buildyear']] = build_year_area['factor'] + for index, item in community_factor_df[ + (community_factor_df['total_price'].isna()) & (community_factor_df['finish_building_date'].notna())].iterrows(): finish_building_date = item['finish_building_date'] if finish_building_date < 1980: - community_factor_df.loc[index, 'factor'] = build_year_area_dict[item['city'] + "|" + item['region'] + "|1980以前"] if item['city'] + "|" + item['region'] + "|1980以前" in build_year_area_dict else None + community_factor_df.loc[index, 'factor'] = build_year_area_dict[ + item['city'] + "|" + item['region'] + "|1980以前"] if item['city'] + "|" + item[ + 'region'] + "|1980以前" in build_year_area_dict else None elif 1980 <= finish_building_date < 1990: - community_factor_df.loc[index, 'factor'] = build_year_area_dict[item['city'] + "|" + item['region'] + "|[1980-1990)"] if item['city'] + "|" + item['region'] + "|[1980-1990)" in build_year_area_dict else None + community_factor_df.loc[index, 'factor'] = build_year_area_dict[ + item['city'] + "|" + item['region'] + "|[1980-1990)"] if item['city'] + "|" + item[ + 'region'] + "|[1980-1990)" in build_year_area_dict else None elif 1990 <= finish_building_date < 2000: - community_factor_df.loc[index, 'factor'] = build_year_area_dict[item['city'] + "|" + item['region'] + "|[1990-2000)"] if item['city'] + "|" + item['region'] + "|[1990-2000)" in build_year_area_dict else None + community_factor_df.loc[index, 'factor'] = build_year_area_dict[ + item['city'] + "|" + item['region'] + "|[1990-2000)"] if item['city'] + "|" + item[ + 'region'] + "|[1990-2000)" in build_year_area_dict else None elif 2000 <= finish_building_date < 2010: - community_factor_df.loc[index, 'factor'] = build_year_area_dict[item['city'] + "|" + item['region'] + "|[2000-2010)"] if item['city'] + "|" + item['region'] + "|[2000-2010)" in build_year_area_dict else None + community_factor_df.loc[index, 'factor'] = build_year_area_dict[ + item['city'] + "|" + item['region'] + "|[2000-2010)"] if item['city'] + "|" + item[ + 'region'] + "|[2000-2010)" in build_year_area_dict else None elif 2010 <= finish_building_date < 2020: - community_factor_df.loc[index, 'factor'] = build_year_area_dict[item['city'] + "|" + item['region'] + "|[2010-2020)"] if item['city'] + "|" + item['region'] + "|[2010-2020)" in build_year_area_dict else None + community_factor_df.loc[index, 'factor'] = build_year_area_dict[ + item['city'] + "|" + item['region'] + "|[2010-2020)"] if item['city'] + "|" + item[ + 'region'] + "|[2010-2020)" in build_year_area_dict else None elif finish_building_date >= 2020: - community_factor_df.loc[index, 'factor'] = build_year_area_dict[item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item['region'] + "|2020及以后" in build_year_area_dict else None + community_factor_df.loc[index, 'factor'] = build_year_area_dict[ + item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item[ + 'region'] + "|2020及以后" in build_year_area_dict else None community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] - community_factor_df = community_factor_df[community_factor_df['total_price'].notna()] - community_factor_df['total_price'] = community_factor_df['total_price'].astype(int) + +def complete_by_region_area(community_factor_df): + region_area_df = pd.read_excel(path + '/全国各区户均面积已去极值.xlsx') + region_area_dict = dict() + for index, region_area in region_area_df.iterrows(): + region_area_dict[region_area['city'] + '|' + region_area['region']] = region_area['factor'] + for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows(): + community_factor_df.loc[index, 'factor'] = region_area_dict[item['city'] + "|" + item['region']] if item['city'] + "|" + item['region'] in region_area_dict else None + community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] + + +def sync(): + log.info(f"开始补全户均总价信息至206 {index_name}") + + community_df = es206_client.scan_by_sql_to_data_frame("select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null") + + community_factor_df = complete_by_area(community_df) + complete_by_build_year_area(community_factor_df) + complete_by_region_area(community_factor_df) save(community_factor_df)