diff --git a/.airflowignore b/.airflowignore index e69de29..4fe6a90 100644 --- a/.airflowignore +++ b/.airflowignore @@ -0,0 +1 @@ +poi/business/* \ No newline at end of file diff --git a/.gitignore b/.gitignore index d2c55d6..929f160 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,2 @@ /venv/ -.idea +.idea \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..415b0d8 --- /dev/null +++ b/README.md @@ -0,0 +1,8 @@ +##### 1. 依赖安装 +pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/ + +##### 2. 具体业务逻辑 + +- 具体业务逻辑放入xxx/business中,如poi/business +- 添加.airflowignore xxx/business/*, 如poi/business/* +- dag放在xxx目录下即可,命名为xx_dag.py \ No newline at end of file diff --git a/dags/poi/__init__.py b/dags/poi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/poi/business/__init__.py b/dags/poi/business/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/dags/poi/business/district_bank.py b/dags/poi/business/district_bank.py new file mode 100644 index 0000000..c46dc05 --- /dev/null +++ b/dags/poi/business/district_bank.py @@ -0,0 +1,106 @@ +""" +小区周边1公里银行入库业务逻辑 +""" + +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +import pandas as pd +from pymongo import UpdateOne + +elastic_client = ElasticSearchClient(host='192.168.0.206') +district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') + +# 银行code +bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "160106", "160107", "160108", "160109", + "160110", + "160111", "160112", "160113", "160114", "160115", "160117", "160118", "160119", "160120", "160121", + "160122", + "160123", "160124", "160125", "160126", "160127", "160128", "160129", "160130", "160131", "160132", + "160133", + "160134", "160135", "160136", "160137", "160138", "160139", "160140", "160141", "160142", "160143", + "160144", + "160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"] + +city = '临沂' +seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) +seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 +seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + + +def find_around_by_location(city, location): + """ + 查找周边poi + :param city: + :param location: + :return: + """ + bank_df = elastic_client.scan_to_data_frame('amap_poi', { + "_source": ["name"], + "query": { + "bool": { + "must": [ + {"terms": { + "typecode": bank_typecode + }}, + { + "term": { + "cityname": { + "value": city + } + } + }, + { + "term": { + "status": { + "value": 0 + } + } + } + ], + "filter": [ + { + "geo_distance": { + "distance": 1000, + "location": location + } + } + ] + } + } + }); + if len(bank_df) > 0: + return bank_df['name'].tolist() + return [] + + +def deal(city, lng, lat): + print('开始查找' + city + '周边银行') + if lng != None: + location = [lng, lat] + bank_df = find_around_by_location(city, location) + return bank_df + return [] + + +seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1) + +actions = list() +print(city + '周边银行开始入库') +for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, {'$set': {'bank': item['bank'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + district_facilitys_conn.bulk_write(actions) + actions.clear() +if len(actions) > 0: + district_facilitys_conn.bulk_write(actions) + actions.clear() +print(city + '周边银行入库完毕') diff --git a/dags/poi/district_bank_dag.py b/dags/poi/district_bank_dag.py new file mode 100644 index 0000000..00b265d --- /dev/null +++ b/dags/poi/district_bank_dag.py @@ -0,0 +1,22 @@ +""" +小区周边配套DAG +""" +from airflow.operators.bash import BashOperator + +import pendulum + +from airflow import DAG + + +with DAG( + "district_bank_dag", + schedule_interval="@monthly", + start_date=pendulum.datetime(2022, 6, 18, tz="UTC"), + catchup=True, +) as dag: + t1 = BashOperator( + task_id='district_bank', + bash_command='business/district_bank', + ) + t1 + diff --git a/requirements.txt b/requirements.txt index 067fa52..59c36ff 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ -apache-airflow \ No newline at end of file +apache-airflow +goojia_utils \ No newline at end of file