优化写入中间表

This commit is contained in:
Jian Shen
2023-03-18 13:13:58 +08:00
parent 94cf7da75d
commit 2654fcfe67
+5 -5
View File
@@ -113,6 +113,11 @@ def sync(city=None):
seaweed_existed_df = pd.DataFrame()
seaweed_not_existed_df = seaweed_df
# 写入csv文件,防止小区变动导致的后续过程更新报错,
seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入中间表clickhouse,解决多worker节点问题
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
actions = list()
# 对于存在的更新
count = 0
@@ -162,11 +167,6 @@ def sync(city=None):
log.info("插入:" + str(count))
actions.clear()
# 写入csv文件,防止小区变动导致的后续过程更新报错,
#seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False)
# 写入中间表clickhouse,解决多worker节点问题
tmp_ck.to_sql_plus(seaweed_df, 'community_summary')
deal_comm(community_summary_df, seaweed_df)
log.info(f"完成同步小区基本属性至206 {index_name}")