This commit is contained in:
Jian Shen
2022-08-22 18:15:32 +08:00
parent 15238675a2
commit b44842e014
+11 -10
View File
@@ -23,7 +23,6 @@ def sync_district_office(type):
for city in cities:
actions = list()
id_list = list()
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
.batch_size(10000):
@@ -50,16 +49,18 @@ def sync_district_office(type):
seaweed_didata_conn.bulk_write(actions)
actions.clear()
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))[['_id']]
id_df = pd.DataFrame(id_list)
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
if len(didata_id_df) > 0:
didata_id_df = didata_id_df[['_id']]
id_df = pd.DataFrame(id_list)
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
actions.append(action)
if len(actions) == 10000:
seaweed_didata_conn.bulk_write(actions)
actions.clear()
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
actions.append(action)
if len(actions) == 10000:
seaweed_didata_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
seaweed_didata_conn.bulk_write(actions)