新增同步字段
This commit is contained in:
@@ -61,6 +61,28 @@ def build_doc(item):
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
if pd.notna(lng) and pd.notna(lat):
|
||||
doc_dict['location'] = {'lon': lng, 'lat': lat}
|
||||
|
||||
# 新增字段存储
|
||||
all_area = commutils.format_all_area(item['all_area'])
|
||||
if pd.notna(all_area):
|
||||
doc_dict['all_area'] = int(all_area)
|
||||
build_area = commutils.format_build_area(item['build_area'])
|
||||
if pd.notna(build_area):
|
||||
doc_dict['build_area'] = int(build_area)
|
||||
greening_rate = commutils.format_greening_rate(item['virescence_percent'])
|
||||
if pd.notna(greening_rate):
|
||||
doc_dict['greening_rate'] = greening_rate
|
||||
volume_rate = commutils.format_plot_ratio(item['cubage_percent'])
|
||||
if pd.notna(volume_rate):
|
||||
doc_dict['volume_rate'] = volume_rate
|
||||
house_type_area_dic = commutils.format_house_type_area(item['main_type'], item['main_type_area'])
|
||||
main_house_type = house_type_area_dic['house_type']
|
||||
main_house_area = house_type_area_dic['house_area']
|
||||
if pd.notna(main_house_area):
|
||||
doc_dict['main_house_area'] = int(float(main_house_area))
|
||||
if pd.notna(main_house_type):
|
||||
doc_dict['main_house_type'] = main_house_type
|
||||
|
||||
return doc_dict
|
||||
|
||||
|
||||
@@ -98,7 +120,9 @@ def sync(city=None):
|
||||
list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1,
|
||||
'alias': 1, 'estate_type2': 1, 'build_type': 1,
|
||||
'household_count': 1, 'block': 1,
|
||||
'estate_charge': 1, 'finish_building_date': 1})
|
||||
'estate_charge': 1, 'finish_building_date': 1,
|
||||
'all_area': 1, 'build_area': 1, 'main_type': 1, 'main_type_area': 1, 'cubage_percent': 1, 'virescence_percent': 1
|
||||
})
|
||||
.batch_size(100000)))
|
||||
|
||||
sql = f"select id from {index_name} "
|
||||
@@ -126,9 +150,10 @@ def sync(city=None):
|
||||
seaweed_not_existed_df = seaweed_df
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错,
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
# seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
# 写入中间表clickhouse,解决多worker节点问题
|
||||
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
|
||||
tmp_ck.to_sql_plus(seaweed_df[['city', 'region', 'name', 'lng', 'lat', 'alias', 'estate_type2', 'build_type',
|
||||
'household_count', 'block', 'estate_charge', 'finish_building_date']], 'community_summary')
|
||||
|
||||
actions = list()
|
||||
# 对于存在的更新
|
||||
|
||||
Reference in New Issue
Block a user