From db2b2005dd1b12fcbf317569546140f0809ad1ef Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 25 Apr 2023 13:41:10 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=90=8C=E6=AD=A5=E5=AD=97?= =?UTF-8?q?=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/comm.py | 31 +++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index 0ff9cc0..4d81b80 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -61,6 +61,28 @@ def build_doc(item): lat = commutils.format_lat_lng(item['lat']) if pd.notna(lng) and pd.notna(lat): doc_dict['location'] = {'lon': lng, 'lat': lat} + + # 新增字段存储 + all_area = commutils.format_all_area(item['all_area']) + if pd.notna(all_area): + doc_dict['all_area'] = int(all_area) + build_area = commutils.format_build_area(item['build_area']) + if pd.notna(build_area): + doc_dict['build_area'] = int(build_area) + greening_rate = commutils.format_greening_rate(item['virescence_percent']) + if pd.notna(greening_rate): + doc_dict['greening_rate'] = greening_rate + volume_rate = commutils.format_plot_ratio(item['cubage_percent']) + if pd.notna(volume_rate): + doc_dict['volume_rate'] = volume_rate + house_type_area_dic = commutils.format_house_type_area(item['main_type'], item['main_type_area']) + main_house_type = house_type_area_dic['house_type'] + main_house_area = house_type_area_dic['house_area'] + if pd.notna(main_house_area): + doc_dict['main_house_area'] = int(float(main_house_area)) + if pd.notna(main_house_type): + doc_dict['main_house_type'] = main_house_type + return doc_dict @@ -98,7 +120,9 @@ def sync(city=None): list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, 'alias': 1, 'estate_type2': 1, 'build_type': 1, 'household_count': 1, 'block': 1, - 'estate_charge': 1, 'finish_building_date': 1}) + 'estate_charge': 1, 'finish_building_date': 1, + 'all_area': 1, 'build_area': 1, 'main_type': 1, 'main_type_area': 1, 'cubage_percent': 1, 'virescence_percent': 1 + }) .batch_size(100000))) sql = f"select id from {index_name} " @@ -126,9 +150,10 @@ def sync(city=None): seaweed_not_existed_df = seaweed_df # 写入csv文件,防止小区变动导致的后续过程更新报错, - seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) + # seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) # 写入中间表clickhouse,解决多worker节点问题 - tmp_ck.to_sql_plus(seaweed_df, 'community_summary') + tmp_ck.to_sql_plus(seaweed_df[['city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type', + 'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary') actions = list() # 对于存在的更新