From 9a1f6a988e47930607dd56470a814bdd4fad15ee Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 20 Jun 2022 09:40:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9=E5=8C=BB?= =?UTF-8?q?=E9=99=A2=E5=85=A5=E5=BA=93?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/poi/business/district_hosoital.py | 105 +++++++++++++++++++++++++ dags/poi/district_bank_dag.py | 3 +- dags/poi/district_bus_dag.py | 3 +- dags/poi/district_hospital_dag.py | 22 ++++++ dags/poi/district_supermarket_dag.py | 3 +- 5 files changed, 133 insertions(+), 3 deletions(-) create mode 100644 dags/poi/business/district_hosoital.py create mode 100644 dags/poi/district_hospital_dag.py diff --git a/dags/poi/business/district_hosoital.py b/dags/poi/business/district_hosoital.py new file mode 100644 index 0000000..8bf8ea7 --- /dev/null +++ b/dags/poi/business/district_hosoital.py @@ -0,0 +1,105 @@ +""" +小区周边1公里医院入库业务逻辑 +""" +import logging +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +import pandas as pd +from pymongo import UpdateOne + +log = logging.getLogger(__name__) + +elastic_client = ElasticSearchClient(host='192.168.0.206') +district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') + +city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1})) + +# 医院code +hospital_typecode = ["090100", "090101", "090200"] + +log.info("开始执行周边医院配套入库") + +def find_around_by_location(city, location): + """ + 查找周边poi + :param city: + :param location: + :return: + """ + try: + hospital_df = elastic_client.scan_to_data_frame('amap_poi', { + "_source": ["name"], + "query": { + "bool": { + "must_not": [ + { + "term": { + "status": { + "value": 1 + } + } + } + ], + "filter": [ + {"terms": { + "typecode": hospital_typecode + }}, + { + "term": { + "cityname": { + "value": city + } + } + }, + { + "geo_distance": { + "distance": 1000, + "location": location + } + } + ] + } + } + }); + if len(hospital_df) > 0: + return hospital_df['name'].tolist() + return [] + except Exception as e: + print(e) + return [] + +def deal(city, lng, lat): + location = [lng, lat] + hospital_df = find_around_by_location(city, location) + return hospital_df + +for city_item in city_dict: + city = city_item['display'] + log.info('开始查找' + city + '周边医院') + seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) + if len(seaweed_df) <= 0: + continue + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + + actions = list() + log.info(city + '周边医院开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, {'$set': {'hospital': item['hospital'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + district_facilitys_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + district_facilitys_conn.bulk_write(actions) + actions.clear() + log.info(city + '周边医院入库完毕') diff --git a/dags/poi/district_bank_dag.py b/dags/poi/district_bank_dag.py index 8ecf6b8..a41d023 100644 --- a/dags/poi/district_bank_dag.py +++ b/dags/poi/district_bank_dag.py @@ -9,7 +9,8 @@ from airflow import DAG with DAG( "district_bank_dag", - schedule_interval="@monthly", + # 每月1日1点运行 + schedule_interval="0 0 1 1 * ?", start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"), catchup=True, ) as dag: diff --git a/dags/poi/district_bus_dag.py b/dags/poi/district_bus_dag.py index 83686ac..7cff321 100644 --- a/dags/poi/district_bus_dag.py +++ b/dags/poi/district_bus_dag.py @@ -9,7 +9,8 @@ from airflow import DAG with DAG( "district_bus_dag", - schedule_interval="@monthly", + # 每月1日9点运行 + schedule_interval="0 0 9 1 * ?", start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"), catchup=True, ) as dag: diff --git a/dags/poi/district_hospital_dag.py b/dags/poi/district_hospital_dag.py new file mode 100644 index 0000000..cf14607 --- /dev/null +++ b/dags/poi/district_hospital_dag.py @@ -0,0 +1,22 @@ +""" +小区周边银行DAG +""" +from airflow.operators.bash import BashOperator + +import pendulum + +from airflow import DAG + +with DAG( + "district_hospital_dag", + # 每月1日17点运行 + schedule_interval="0 0 17 1 * ?", + start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"), + catchup=True, +) as dag: + t1 = BashOperator( + task_id='district_hospital', + bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_hospital.py', + ) + t1 + diff --git a/dags/poi/district_supermarket_dag.py b/dags/poi/district_supermarket_dag.py index 9215ae1..5b1968f 100644 --- a/dags/poi/district_supermarket_dag.py +++ b/dags/poi/district_supermarket_dag.py @@ -9,7 +9,8 @@ from airflow import DAG with DAG( "district_supermarket_dag", - schedule_interval="@monthly", + # 每月2日1点运行 + schedule_interval="0 0 1 2 * ?", start_date=pendulum.datetime(2022, 6, 18, tz="Asia/Shanghai"), catchup=True, ) as dag: