From f9fbb840653597929cd7c72ecab12e2b90ff7c97 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 22 Aug 2022 15:36:15 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA43=E5=90=8C?= =?UTF-8?q?=E6=AD=A5=E8=87=B3100=E9=AB=98=E6=95=88=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/didata_sync.py | 70 +++++++++++++++++++++++++++ dags/business/didata_sync_dag.py | 17 +++++++ 2 files changed, 87 insertions(+) create mode 100644 dags/business/business/didata_sync.py create mode 100644 dags/business/didata_sync_dag.py diff --git a/dags/business/business/didata_sync.py b/dags/business/business/didata_sync.py new file mode 100644 index 0000000..2fbff99 --- /dev/null +++ b/dags/business/business/didata_sync.py @@ -0,0 +1,70 @@ +import logging +import sys +from goojia_utils import MAIN_MONGO_CLIENT, DIDATA_MONGO_CLIENT, areautils +from pymongo import UpdateOne +from bson.objectid import ObjectId +import pandas as pd +import time + +log = logging.getLogger(__name__) + +type = sys.argv[1] if len(sys.argv) > 1 else '' +log.info("同步类型:" + type) + + +def sync_district_office(type): + if type != 'district' and type != 'office': + return + seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + seaweed_didata_conn = DIDATA_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + + cities = areautils.get_all_cities() + update_date = int(round(time.time() * 1000)) + for city in cities: + actions = list() + id_list = list() + + for doc in seaweed_conn.find({'city': city, 'cat': type, 'status': 0}, {'city': 1, 'region': 1, 'block': 1, 'cat': 1, 'name': 1, 'address': 1, + 'estate_type2': 1, 'township': 1, 'alias': 1, 'addr_alias': 1, 'lng': 1, 'lat': 1, 'status': 1})\ + .batch_size(10000): + updateDict = {'city': doc['city'], 'region': doc['region'], 'category': doc['cat'], 'name': doc['name'], + 'estate_type': doc['estate_type2'], 'status': doc['status']} + id_list.append({'_id': doc['_id']}) + if 'block' in doc: + updateDict['block'] = doc['block'] + if 'address' in doc: + updateDict['address'] = doc['address'] + if 'township' in doc: + updateDict['township'] = doc['township'] + if 'alias' in doc: + updateDict['alias'] = doc['alias'] + if 'addr_alias' in doc: + updateDict['addr_alias'] = doc['addr_alias'] + if 'lng' in doc and str(doc['lng']) != 'nan': + updateDict['location'] = {'lon': doc['lng'] / 10**10, 'lat': doc['lat'] / 10**10} + updateDict['update_date'] = update_date + action = UpdateOne({'_id': doc['_id']}, {'$set': updateDict}, upsert=True) + + actions.append(action) + if len(actions) == 10000: + seaweed_didata_conn.bulk_write(actions) + actions.clear() + + didata_id_df = pd.DataFrame(list(seaweed_didata_conn.find({'city': city, 'cat': type}, {'_id': 1}).batch_size(10000)))[['_id']] + id_df = pd.DataFrame(id_list) + deleted_seaweed_df = pd.concat([didata_id_df, id_df, id_df]).drop_duplicates(keep=False) + + for deleted_seaweed_index, deleted_seaweed_item in deleted_seaweed_df.iterrows(): + action = UpdateOne({'_id': ObjectId(deleted_seaweed_item['_id'])}, {'$set': {'status': 1, 'update_date': update_date}}, upsert=False) + actions.append(action) + if len(actions) == 10000: + seaweed_didata_conn.bulk_write(actions) + actions.clear() + + if len(actions) > 0: + seaweed_didata_conn.bulk_write(actions) + actions.clear() + log.info(city + "-" + type + "-同步完成") + + +sync_district_office(type) \ No newline at end of file diff --git a/dags/business/didata_sync_dag.py b/dags/business/didata_sync_dag.py new file mode 100644 index 0000000..d86c6c6 --- /dev/null +++ b/dags/business/didata_sync_dag.py @@ -0,0 +1,17 @@ +from airflow.operators.bash import BashOperator +import pendulum +from airflow import DAG + +with DAG( + "didata_sync_dag", + # 每月28日3点整分运行 + schedule_interval="0 3 28 * *", + start_date=pendulum.datetime(2022, 8, 22, tz="Asia/Shanghai"), + catchup=True, + description="地数相关数据同步type: district或office" +) as dag: + t1 = BashOperator( + task_id='didata_sync_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/didata_sync.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t1 \ No newline at end of file