Files
airflow/dags/poi/business/district_school.py
T
2025-02-17 14:55:00 +08:00

107 lines
5.4 KiB
Python

"""
小区周边1公里教育入库业务逻辑
"""
import logging
from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread
import pandas as pd
from pymongo import UpdateOne
import poiutils
log = logging.getLogger(__name__)
district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys')
surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities')
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping')
# 教育code
school_typecode = ["141200", "141201", "141202", "141203", "141204"]
small_type = ["学校", "高等院校", "中学", "小学", "幼儿园"]
def deal3(item):
location = [item['lng'], item['lat']]
item['school'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 3000)
item['kindergarten'] = poiutils.find_around_by_location(item['city_display'], location, ["幼儿园"], 3000)
item['primary_school'] = poiutils.find_around_by_location(item['city_display'], location, ["小学"], 3000)
item['middle_school'] = poiutils.find_around_by_location(item['city_display'], location, ["中学"], 3000)
return item
def deal(item):
location = [item['lng'], item['lat']]
item['school'] = poiutils.find_around_by_location(item['city_display'], location, small_type, 1000)
item['kindergarten'] = poiutils.find_around_by_location(item['city_display'], location, ["幼儿园"], 1000)
item['primary_school'] = poiutils.find_around_by_location(item['city_display'], location, ["小学"], 1000)
item['middle_school'] = poiutils.find_around_by_location(item['city_display'], location, ["中学"], 1000)
return item
def do():
sync_new()
sync1()
def sync_new(distance=3000):
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['city_display'] = city_display
seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30)
#seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1)
actions = list()
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'school': item['school'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
surrounding_facilities_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
surrounding_facilities_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边教育入库完毕')
def sync1():
city_dict = list(administrative_mapping_conn.find({'type': '城市'}))
for city_item in city_dict:
city = city_item['city_short']
city_display = city_item['city']
seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}},
{'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000)))
if len(seaweed_df) <= 0:
continue
seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10
seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10
seaweed_df['city_display'] = city_display
seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30)
#seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1)
actions = list()
for index, item in seaweed_df.iterrows():
try:
# 周边配套库中历史原因,id可能不为小区id
action = UpdateOne({'city': item['city'], 'region': item['region'],
'district_name': item['name']}, {'$set': {'school': item['school'], 'city': item['city'], 'region': item['region'],
'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True)
actions.append(action)
except Exception as e:
print(e)
if len(actions) == 5000:
district_facilitys_conn.bulk_write(actions)
actions.clear()
if len(actions) > 0:
district_facilitys_conn.bulk_write(actions)
actions.clear()
log.info(city + '周边教育入库完毕')