diff --git a/dags/business/business/didata_sync.py b/dags/business/business/didata_sync.py index 3ffe5b3..190d6dd 100644 --- a/dags/business/business/didata_sync.py +++ b/dags/business/business/didata_sync.py @@ -23,7 +23,6 @@ def sync_district_office(type): for city in cities: actions = list() id_list = list() - for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\ .batch_size(10000): @@ -50,16 +49,18 @@ def sync_district_office(type): seaweed_didata_conn.bulk_write(actions) actions.clear() - didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000)))[['_id']] - id_df = pd.DataFrame(id_list) - deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False) + didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000))) + if len(didata_id_df) > 0: + didata_id_df = didata_id_df[['_id']] + id_df = pd.DataFrame(id_list) + deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False) - for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows(): - action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) - actions.append(action) - if len(actions) == 10000: - seaweed_didata_conn.bulk_write(actions) - actions.clear() + for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows(): + action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) + actions.append(action) + if len(actions) == 10000: + seaweed_didata_conn.bulk_write(actions) + actions.clear() if len(actions) > 0: seaweed_didata_conn.bulk_write(actions)