优化写入中间表

This commit is contained in:
Jian Shen
2023-03-18 14:14:27 +08:00
parent 8406955a7e
commit d8543eae5b
+9 -1
View File
@@ -103,6 +103,14 @@ def sync(city=None):
sql += f" where city='{city}'"
community_summary_df = es206_client.scan_by_sql_to_data_frame(sql)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
# 清除别名中逗号
def dealAlias(x):
if isinstance(x, list) and len(x) > 0:
for index, element in enumerate(x):
if "'" in element:
x[index] = str(element).replace("'", "")
return x
seaweed_df['alias'] = seaweed_df['alias'].apply(lambda x: dealAlias(x))
if len(community_summary_df) > 0:
community_summary_df['flag'] = 1
@@ -114,7 +122,7 @@ def sync(city=None):
seaweed_not_existed_df = seaweed_df
# 写入csv文件,防止小区变动导致的后续过程更新报错,
#seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入中间表clickhouse,解决多worker节点问题
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')