新增周边教育

This commit is contained in:
Jian Shen
2022-06-20 09:55:11 +08:00
parent e381b71da1
commit 4da63ac0c9
2 changed files with 127 additions and 0 deletions
+105
View File
@@ -0,0 +1,105 @@
"""
小区周边1公里教育入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient
import pandas as pd
from pymongo import UpdateOne
log = logging.getLogger(__name__)
elastic_client = ElasticSearchClient(host='192.168.0.206')
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area')
city_dict = list(server_area_conn.find({'city_grade': {'$ne': 0}}, {'display': 1}))
# 教育code
school_typecode = ["141200", "141201", "141202", "141203", "141204"]
log.info("开始执行周边教育配套入库")
def find_around_by_location(city, location):
"""
查找周边poi
:param city:
:param location:
:return:
"""
try:
school_df = elastic_client.scan_to_data_frame('amap_poi', {
"_source": ["name"],
"query": {
"bool": {
"must_not": [
{
"term": {
"status": {
"value": 1
}
}
}
],
"filter": [
{"terms": {
"typecode": school_typecode
}},
{
"term": {
"cityname": {
"value": city
}
}
},
{
"geo_distance": {
"distance": 1000,
"location": location
}
}
]
}
}
});
if len(school_df) > 0:
return school_df['name'].tolist()
return []
except Exception as e:
print(e)
return []
def deal(city, lng, lat):
location = [lng, lat]
school_df = find_around_by_location(city, location)
return school_df
for city_item in city_dict:
city = city_item['display']
log.info('开始查找' + city + '周边教育')
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['school'] = seaweed_df.apply(lambda x: deal(x['city'], x['lng'], x['lat']), axis=1)
actions = list()
log.info(city + '周边教育开始入库')
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'school': item['school'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边教育入库完毕')
+22
View File
@@ -0,0 +1,22 @@
"""
小区周边银行DAG
"""
from airflow.operators.bash import BashOperator
import pendulum
from airflow import DAG
with DAG(
"district_school_dag",
# 每月2日9点运行
schedule_interval="0 9 2 * *",
start_date=pendulum.datetime(2022, 6, 20, tz="Asia/Shanghai"),
catchup=True,
) as dag:
t1 = BashOperator(
task_id='district_school',
bash_command='python /opt/airflow/dags/repo/dags/poi/business/district_school.py',
)
t1