新增周边配套
This commit is contained in:
@@ -0,0 +1 @@
|
|||||||
|
poi/business/*
|
||||||
@@ -0,0 +1,8 @@
|
|||||||
|
##### 1. 依赖安装
|
||||||
|
pip install -r requirements.txt -i https://repo.fangjia.com/repository/pypi-public/simple/
|
||||||
|
|
||||||
|
##### 2. 具体业务逻辑
|
||||||
|
|
||||||
|
- 具体业务逻辑放入xxx/business中,如poi/business
|
||||||
|
- 添加.airflowignore xxx/business/*, 如poi/business/*
|
||||||
|
- dag放在xxx目录下即可,命名为xx_dag.py
|
||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""
|
||||||
|
小区周边1公里银行入库业务逻辑
|
||||||
|
"""
|
||||||
|
|
||||||
|
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient
|
||||||
|
import pandas as pd
|
||||||
|
from pymongo import UpdateOne
|
||||||
|
|
||||||
|
elastic_client = ElasticSearchClient(host='192.168.0.206')
|
||||||
|
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
|
||||||
|
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||||
|
|
||||||
|
# 银行code
|
||||||
|
bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "160106", "160107", "160108", "160109",
|
||||||
|
"160110",
|
||||||
|
"160111", "160112", "160113", "160114", "160115", "160117", "160118", "160119", "160120", "160121",
|
||||||
|
"160122",
|
||||||
|
"160123", "160124", "160125", "160126", "160127", "160128", "160129", "160130", "160131", "160132",
|
||||||
|
"160133",
|
||||||
|
"160134", "160135", "160136", "160137", "160138", "160139", "160140", "160141", "160142", "160143",
|
||||||
|
"160144",
|
||||||
|
"160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"]
|
||||||
|
|
||||||
|
city = '临沂'
|
||||||
|
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0},
|
||||||
|
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(
|
||||||
|
10000)))
|
||||||
|
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
|
||||||
|
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
|
||||||
|
|
||||||
|
|
||||||
|
def find_around_by_location(city, location):
|
||||||
|
"""
|
||||||
|
查找周边poi
|
||||||
|
:param city:
|
||||||
|
:param location:
|
||||||
|
:return:
|
||||||
|
"""
|
||||||
|
bank_df = elastic_client.scan_to_data_frame('amap_poi', {
|
||||||
|
"_source": ["name"],
|
||||||
|
"query": {
|
||||||
|
"bool": {
|
||||||
|
"must": [
|
||||||
|
{"terms": {
|
||||||
|
"typecode": bank_typecode
|
||||||
|
}},
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"cityname": {
|
||||||
|
"value": city
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"term": {
|
||||||
|
"status": {
|
||||||
|
"value": 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"filter": [
|
||||||
|
{
|
||||||
|
"geo_distance": {
|
||||||
|
"distance": 1000,
|
||||||
|
"location": location
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if len(bank_df) > 0:
|
||||||
|
return bank_df['name'].tolist()
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def deal(city, lng, lat):
|
||||||
|
print('开始查找' + city + '周边银行')
|
||||||
|
if lng != None:
|
||||||
|
location = [lng, lat]
|
||||||
|
bank_df = find_around_by_location(city, location)
|
||||||
|
return bank_df
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
|
||||||
|
|
||||||
|
actions = list()
|
||||||
|
print(city + '周边银行开始入库')
|
||||||
|
for index, item in seaweed_df.iterrows():
|
||||||
|
try:
|
||||||
|
# 周边配套库中历史原因,id可能不为小区id
|
||||||
|
action = UpdateOne({'city': item['city'], 'region': item['region'],
|
||||||
|
'district_name': item['name']}, {'$set': {'bank': item['bank'], 'city': item['city'], 'region': item['region'],
|
||||||
|
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
|
||||||
|
actions.append(action)
|
||||||
|
except Exception as e:
|
||||||
|
print(e)
|
||||||
|
if len(actions) == 5000:
|
||||||
|
district_facilitys_conn.bulk_write(actions)
|
||||||
|
actions.clear()
|
||||||
|
if len(actions) > 0:
|
||||||
|
district_facilitys_conn.bulk_write(actions)
|
||||||
|
actions.clear()
|
||||||
|
print(city + '周边银行入库完毕')
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
"""
|
||||||
|
小区周边配套DAG
|
||||||
|
"""
|
||||||
|
from airflow.operators.bash import BashOperator
|
||||||
|
|
||||||
|
import pendulum
|
||||||
|
|
||||||
|
from airflow import DAG
|
||||||
|
|
||||||
|
|
||||||
|
with DAG(
|
||||||
|
"district_bank_dag",
|
||||||
|
schedule_interval="@monthly",
|
||||||
|
start_date=pendulum.datetime(2022, 6, 18, tz="UTC"),
|
||||||
|
catchup=True,
|
||||||
|
) as dag:
|
||||||
|
t1 = BashOperator(
|
||||||
|
task_id='district_bank',
|
||||||
|
bash_command='business/district_bank',
|
||||||
|
)
|
||||||
|
t1
|
||||||
|
|
||||||
@@ -1 +1,2 @@
|
|||||||
apache-airflow
|
apache-airflow
|
||||||
|
goojia_utils
|
||||||
Reference in New Issue
Block a user