优化
This commit is contained in:
@@ -23,7 +23,6 @@ def sync_district_office(type):
|
||||
for city in cities:
|
||||
actions = list()
|
||||
id_list = list()
|
||||
|
||||
for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1,
|
||||
'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\
|
||||
.batch_size(10000):
|
||||
@@ -50,16 +49,18 @@ def sync_district_office(type):
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))[['_id']]
|
||||
id_df = pd.DataFrame(id_list)
|
||||
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
||||
didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))
|
||||
if len(didata_id_df) > 0:
|
||||
didata_id_df = didata_id_df[['_id']]
|
||||
id_df = pd.DataFrame(id_list)
|
||||
deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False)
|
||||
|
||||
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows():
|
||||
action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False)
|
||||
actions.append(action)
|
||||
if len(actions) == 10000:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
actions.clear()
|
||||
|
||||
if len(actions) > 0:
|
||||
seaweed_didata_conn.bulk_write(actions)
|
||||
|
||||
Reference in New Issue
Block a user