优化写入中间表
This commit is contained in:
@@ -103,6 +103,14 @@ def sync(city=None):
|
||||
sql += f" where city='{city}'"
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
# 清除别名中逗号
|
||||
def dealAlias(x):
|
||||
if isinstance(x, list) and len(x) > 0:
|
||||
for index, element in enumerate(x):
|
||||
if "'" in element:
|
||||
x[index] = str(element).replace("'", "")
|
||||
return x
|
||||
seaweed_df['alias'] = seaweed_df['alias'].apply(lambda x: dealAlias(x))
|
||||
|
||||
if len(community_summary_df) > 0:
|
||||
community_summary_df['flag'] = 1
|
||||
@@ -114,7 +122,7 @@ def sync(city=None):
|
||||
seaweed_not_existed_df = seaweed_df
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错,
|
||||
#seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
# 写入中间表clickhouse,解决多worker节点问题
|
||||
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
|
||||
|
||||
|
||||
Reference in New Issue
Block a user