diff --git a/dags/business/business/didata_sync.py b/dags/business/business/didata_sync.py index b00a59d..8573fc5 100644 --- a/dags/business/business/didata_sync.py +++ b/dags/business/business/didata_sync.py @@ -1,82 +1,82 @@ -import logging -import sys -from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils -from pymongo import UpdateOne -from bson.objectid import ObjectId -import pandas as pd -import time - -log = logging.getLogger(__name__) - -type = sys.argv[1] if len(sys.argv) > 1 else 'all' -log.info("同步类型:" + type) - - -def sync_district_office(type): - if type != 'district' and type != 'office': - return - seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') - seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed') - - cities = areautils.get_all_cities() - update_date = int(round(time.time() * 1000)) - for city in cities: - actions = list() - id_list = list() - for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, - {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, - 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, - 'status': 1}) \ - .batch_size(10000): - updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], - 'status': doc['status']} - if 'estate_type2' in doc: - updateDict['estate_type'] = doc['estate_type2'] - - id_list.append({'_id': doc['_id']}) - if 'block' in doc: - updateDict['block'] = doc['block'] - if 'address' in doc: - updateDict['address'] = doc['address'] - if 'township' in doc: - updateDict['township'] = doc['township'] - if 'alias' in doc: - updateDict['alias'] = doc['alias'] - if 'addr_alias' in doc: - updateDict['addr_alias'] = doc['addr_alias'] - if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan': - updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10} - updateDict['update_date'] = update_date - action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True) - - actions.append(action) - if len(actions) == 10000: - seaweed_didata_conn.bulk_write(actions) - actions.clear() - - didata_id_df = pd.DataFrame( - list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000))) - if len(didata_id_df) > 0: - didata_id_df = didata_id_df[['_id']] - id_df = pd.DataFrame(id_list) - deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False) - - for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows(): - action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, - {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) - actions.append(action) - if len(actions) == 10000: - seaweed_didata_conn.bulk_write(actions) - actions.clear() - - if len(actions) > 0: - seaweed_didata_conn.bulk_write(actions) - actions.clear() - log.info(city + "-" + type + "-同步完成") - - -if type == 'all': - sync_district_office("district") - sync_district_office("office") -else: - sync_district_office(type) +# import logging +# import sys +# from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils +# from pymongo import UpdateOne +# from bson.objectid import ObjectId +# import pandas as pd +# import time +# +# log = logging.getLogger(__name__) +# +# type = sys.argv[1] if len(sys.argv) > 1 else 'all' +# log.info("同步类型:" + type) +# +# +# def sync_district_office(type): +# if type != 'district' and type != 'office': +# return +# seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +# seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +# +# cities = areautils.get_all_cities() +# update_date = int(round(time.time() * 1000)) +# for city in cities: +# actions = list() +# id_list = list() +# for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, +# {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, +# 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, +# 'status': 1}) \ +# .batch_size(10000): +# updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], +# 'status': doc['status']} +# if 'estate_type2' in doc: +# updateDict['estate_type'] = doc['estate_type2'] +# +# id_list.append({'_id': doc['_id']}) +# if 'block' in doc: +# updateDict['block'] = doc['block'] +# if 'address' in doc: +# updateDict['address'] = doc['address'] +# if 'township' in doc: +# updateDict['township'] = doc['township'] +# if 'alias' in doc: +# updateDict['alias'] = doc['alias'] +# if 'addr_alias' in doc: +# updateDict['addr_alias'] = doc['addr_alias'] +# if 'lng' in doc and doc['lng'] is not None and str(doc['lng']) != 'nan': +# updateDict['location'] = {'lon': doc['lng'] / 10 ** 10, 'lat': doc['lat'] / 10 ** 10} +# updateDict['update_date'] = update_date +# action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True) +# +# actions.append(action) +# if len(actions) == 10000: +# seaweed_didata_conn.bulk_write(actions) +# actions.clear() +# +# didata_id_df = pd.DataFrame( +# list(seaweed_didata_conn.find({'city': city, 'category': type}, {'_id': 1}).batch_size(10000))) +# if len(didata_id_df) > 0: +# didata_id_df = didata_id_df[['_id']] +# id_df = pd.DataFrame(id_list) +# deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False) +# +# for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows(): +# action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, +# {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) +# actions.append(action) +# if len(actions) == 10000: +# seaweed_didata_conn.bulk_write(actions) +# actions.clear() +# +# if len(actions) > 0: +# seaweed_didata_conn.bulk_write(actions) +# actions.clear() +# log.info(city + "-" + type + "-同步完成") +# +# +# if type == 'all': +# sync_district_office("district") +# sync_district_office("office") +# else: +# sync_district_office(type) diff --git a/dags/business/dag_didata_sync.py b/dags/business/dag_didata_sync.py index c23186a..2120659 100644 --- a/dags/business/dag_didata_sync.py +++ b/dags/business/dag_didata_sync.py @@ -2,16 +2,16 @@ from airflow.operators.bash import BashOperator import pendulum from airflow import DAG -with DAG( - dag_id="didata_sync_dag", - # 每月28日3点整分运行 - schedule="0 3 28 * *", - start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), - catchup=True, - description="地数相关数据同步type: district或office, 不填默认同步全部" -) as dag: - t1 = BashOperator( - task_id='didata_sync_dag', - bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', - ) - t1 \ No newline at end of file +# with DAG( +# dag_id="didata_sync_dag", +# # 每月28日3点整分运行 +# schedule="0 3 28 * *", +# start_date=pendulum.datetime(2023, 3, 17, tz="Asia/Shanghai"), +# catchup=True, +# description="地数相关数据同步type: district或office, 不填默认同步全部" +# ) as dag: +# t1 = BashOperator( +# task_id='didata_sync_dag', +# bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', +# ) +# t1 \ No newline at end of file