新增同步字段

This commit is contained in:
shenjian@fangjia.com
2023-04-25 13:41:10 +08:00
parent 29f827e2cc
commit db2b2005dd
+28 -3
View File
@@ -61,6 +61,28 @@ def build_doc(item):
lat = commutils.format_lat_lng(item['lat'])
if pd.notna(lng) and pd.notna(lat):
doc_dict['location'] = {'lon': lng, 'lat': lat}
# 新增字段存储
all_area = commutils.format_all_area(item['all_area'])
if pd.notna(all_area):
doc_dict['all_area'] = int(all_area)
build_area = commutils.format_build_area(item['build_area'])
if pd.notna(build_area):
doc_dict['build_area'] = int(build_area)
greening_rate = commutils.format_greening_rate(item['virescence_percent'])
if pd.notna(greening_rate):
doc_dict['greening_rate'] = greening_rate
volume_rate = commutils.format_plot_ratio(item['cubage_percent'])
if pd.notna(volume_rate):
doc_dict['volume_rate'] = volume_rate
house_type_area_dic = commutils.format_house_type_area(item['main_type'], item['main_type_area'])
main_house_type = house_type_area_dic['house_type']
main_house_area = house_type_area_dic['house_area']
if pd.notna(main_house_area):
doc_dict['main_house_area'] = int(float(main_house_area))
if pd.notna(main_house_type):
doc_dict['main_house_type'] = main_house_type
return doc_dict
@@ -98,7 +120,9 @@ def sync(city=None):
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
'alias': 1, 'estate_type2': 1, 'build_type': 1,
'household_count': 1, 'block': 1,
'estate_charge': 1, 'finish_building_date': 1})
'estate_charge': 1, 'finish_building_date': 1,
'all_area': 1, 'build_area': 1, 'main_type': 1, 'main_type_area': 1, 'cubage_percent': 1, 'virescence_percent': 1
})
.batch_size(100000)))
sql = f"select id from {index_name} "
@@ -126,9 +150,10 @@ def sync(city=None):
seaweed_not_existed_df = seaweed_df
# 写入csv文件,防止小区变动导致的后续过程更新报错,
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入中间表clickhouse,解决多worker节点问题
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
tmp_ck.to_sql_plus(seaweed_df[['city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type',
'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary')
actions = list()
# 对于存在的更新