From b67bd8bd1e88f4114deed5aa47a6e79cab77302c Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Wed, 13 Nov 2024 16:31:35 +0800 Subject: [PATCH] =?UTF-8?q?=E6=94=B9=E9=80=A0=E5=91=A8=E8=BE=B9=E9=85=8D?= =?UTF-8?q?=E5=A5=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/poi/business/district_bank.py | 104 +++++++++---------- dags/poi/business/district_bus.py | 120 ++++++++++------------ dags/poi/business/district_hospital.py | 102 +++++++++--------- dags/poi/business/district_school.py | 95 ++++++++--------- dags/poi/business/district_subway.py | 96 ++++++++--------- dags/poi/business/district_supermarket.py | 98 ++++++++---------- dags/poi/business/poiutils.py | 54 ++++++++++ 7 files changed, 335 insertions(+), 334 deletions(-) create mode 100644 dags/poi/business/poiutils.py diff --git a/dags/poi/business/district_bank.py b/dags/poi/business/district_bank.py index c3ca178..21f6ebc 100644 --- a/dags/poi/business/district_bank.py +++ b/dags/poi/business/district_bank.py @@ -2,14 +2,16 @@ 小区周边1公里银行入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT import pandas as pd from pymongo import UpdateOne +import poiutils + log = logging.getLogger(__name__) -elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') @@ -24,63 +26,55 @@ bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "16 "160144", "160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"] -def find_around_by_location(city, location): - """ - 查找周边poi - :param city: - :param location: - :return: - """ - try: - bank_df = elastic_client.scan_to_data_frame('amap_poi', { - "_source": ["name"], - "query": { - "bool": { - "must_not": [ - { - "term": { - "status": { - "value": 1 - } - } - } - ], - "filter": [ - {"terms": { - "typecode": bank_typecode - }}, - { - "term": { - "cityname": { - "value": city - } - } - }, - { - "geo_distance": { - "distance": 1000, - "location": location - } - } - ] - } - } - }); - if len(bank_df) > 0: - return bank_df['name'].tolist() - return [] - except Exception as e: - print(e) - return [] - -def deal(city, lng, lat): +def deal(city, lng, lat, distance=1000): location = [lng, lat] - bank_df = find_around_by_location(city, location) - return bank_df - + return poiutils.find_around_by_location(city, location, bank_typecode, distance) def do(): + sync1() + sync_new() + +def sync_new(distance=3000): + """ + 周边3公里POI入库 + """ + log.info("开始执行周边银行配套入库") + city_dict = list(administrative_mapping_conn.find({'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] + log.info('开始查找' + city + '周边银行') + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) + if len(seaweed_df) <= 0: + continue + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + + actions = list() + log.info(city + '周边银行开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, + {'$set': {'bank': item['bank'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, + upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + log.info(city + '周边银行入库完毕') + +def sync1(): log.info("开始执行周边银行配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: diff --git a/dags/poi/business/district_bus.py b/dags/poi/business/district_bus.py index 232e0ec..73e2b8f 100644 --- a/dags/poi/business/district_bus.py +++ b/dags/poi/business/district_bus.py @@ -2,14 +2,15 @@ 小区周边1公里交通地铁入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT import pandas as pd from pymongo import UpdateOne +import poiutils log = logging.getLogger(__name__) -elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') @@ -17,71 +18,62 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' bus_typecode = ["150700"] -def find_around_by_location(city, location): - """ - 查找周边poi - :param city: - :param location: - :return: - """ - try: - bus_df = elastic_client.scan_to_data_frame('amap_poi', { - "_source": ["name"], - "query": { - "bool": { - "must_not": [ - { - "term": { - "status": { - "value": 1 - } - } - } - ], - "filter": [ - {"terms": { - "typecode": bus_typecode - }}, - { - "term": { - "cityname": { - "value": city - } - } - }, - { - "geo_distance": { - "distance": 1000, - "location": location - } - } - ] - } - } - }); - if len(bus_df) > 0: - return bus_df['name'].tolist() - return [] - except Exception as e: - print(e) - return [] - - -def deal(city, lng, lat): +def deal(city, lng, lat, distance=1000): location = [lng, lat] - bus_df = find_around_by_location(city, location) - return bus_df - + return poiutils.find_around_by_location(city, location, bus_typecode, distance) def do(): + sync1() + sync_new() + + +def sync_new(distance=3000): log.info("开始执行周边交通配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] log.info('开始查找' + city + '周边交通') - seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, - {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( - 10000))) + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) + if len(seaweed_df) <= 0: + continue + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + + actions = list() + log.info(city + '周边交通开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, + {'$set': {'bus': item['bus'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, + upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + log.info(city + '周边交通入库完毕') + +def sync1(): + log.info("开始执行周边交通配套入库") + city_dict = list(administrative_mapping_conn.find({'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] + log.info('开始查找' + city + '周边交通') + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) if len(seaweed_df) <= 0: continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 @@ -94,8 +86,10 @@ def do(): try: # 周边配套库中历史原因,id可能不为小区id action = UpdateOne({'city': item['city'], 'region': item['region'], - 'district_name': item['name']}, {'$set': {'bus': item['bus'], 'city': item['city'], 'region': item['region'], - 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True) + 'district_name': item['name']}, + {'$set': {'bus': item['bus'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, + upsert=True) actions.append(action) except Exception as e: print(e) @@ -103,6 +97,6 @@ def do(): district_facilitys_conn.bulk_write(actions) actions.clear() if len(actions) > 0: - district_facilitys_conn.bulk_write(actions) - actions.clear() + district_facilitys_conn.bulk_write(actions) + actions.clear() log.info(city + '周边交通入库完毕') diff --git a/dags/poi/business/district_hospital.py b/dags/poi/business/district_hospital.py index 25cc1f3..402c048 100644 --- a/dags/poi/business/district_hospital.py +++ b/dags/poi/business/district_hospital.py @@ -2,14 +2,16 @@ 小区周边1公里医院入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT +import poiutils + import pandas as pd from pymongo import UpdateOne log = logging.getLogger(__name__) -elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') server_area_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'server_area') administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') @@ -18,63 +20,55 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' hospital_typecode = ["090100", "090101", "090200"] -def find_around_by_location(city, location): - """ - 查找周边poi - :param city: - :param location: - :return: - """ - try: - hospital_df = elastic_client.scan_to_data_frame('amap_poi', { - "_source": ["name"], - "query": { - "bool": { - "must_not": [ - { - "term": { - "status": { - "value": 1 - } - } - } - ], - "filter": [ - {"terms": { - "typecode": hospital_typecode - }}, - { - "term": { - "cityname": { - "value": city - } - } - }, - { - "geo_distance": { - "distance": 1000, - "location": location - } - } - ] - } - } - }); - if len(hospital_df) > 0: - return hospital_df['name'].tolist() - return [] - except Exception as e: - print(e) - return [] - - -def deal(city, lng, lat): +def deal(city, lng, lat, distance=1000): location = [lng, lat] - hospital_df = find_around_by_location(city, location) + hospital_df = poiutils.find_around_by_location(city, location, hospital_typecode, distance) return hospital_df def do(): + sync1() + sync_new() + +def sync_new(distance=3000): + log.info("开始执行周边医院配套入库") + city_dict = list(administrative_mapping_conn.find({'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] + log.info('开始查找' + city + '周边医院') + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) + if len(seaweed_df) <= 0: + continue + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + + actions = list() + log.info(city + '周边医院开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, { + '$set': {'hospital': item['hospital'], 'city': item['city'], + 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, + upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + log.info(city + '周边医院入库完毕') + + +def sync1(): log.info("开始执行周边医院配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: diff --git a/dags/poi/business/district_school.py b/dags/poi/business/district_school.py index bec39f1..8c3a2a2 100644 --- a/dags/poi/business/district_school.py +++ b/dags/poi/business/district_school.py @@ -2,14 +2,15 @@ 小区周边1公里教育入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT import pandas as pd from pymongo import UpdateOne +import poiutils log = logging.getLogger(__name__) -elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') @@ -17,62 +18,50 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' school_typecode = ["141200", "141201", "141202", "141203", "141204"] -def find_around_by_location(city, location): - """ - 查找周边poi - :param city: - :param location: - :return: - """ - try: - school_df = elastic_client.scan_to_data_frame('amap_poi', { - "_source": ["name"], - "query": { - "bool": { - "must_not": [ - { - "term": { - "status": { - "value": 1 - } - } - } - ], - "filter": [ - {"terms": { - "typecode": school_typecode - }}, - { - "term": { - "cityname": { - "value": city - } - } - }, - { - "geo_distance": { - "distance": 1000, - "location": location - } - } - ] - } - } - }); - if len(school_df) > 0: - return school_df['name'].tolist() - return [] - except Exception as e: - print(e) - return [] - -def deal(city, lng, lat): +def deal(city, lng, lat, distance=1000): location = [lng, lat] - school_df = find_around_by_location(city, location) + school_df = poiutils.find_around_by_location(city, location, school_typecode, distance) return school_df def do(): + sync1() + sync_new() + +def sync_new(distance=3000): + log.info("开始执行周边教育配套入库") + city_dict = list(administrative_mapping_conn.find({'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] + log.info('开始查找' + city + '周边教育') + seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) + if len(seaweed_df) <= 0: + continue + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + + actions = list() + log.info(city + '周边教育开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, {'$set': {'school': item['school'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + log.info(city + '周边教育入库完毕') + +def sync1(): log.info("开始执行周边教育配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: diff --git a/dags/poi/business/district_subway.py b/dags/poi/business/district_subway.py index db3f085..1cdaceb 100644 --- a/dags/poi/business/district_subway.py +++ b/dags/poi/business/district_subway.py @@ -5,75 +5,63 @@ import logging from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient import pandas as pd from pymongo import UpdateOne +import poiutils log = logging.getLogger(__name__) elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') +surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') - # 地铁code subway_typecode = ["150500"] -def find_around_by_location(city, location): - """ - 查找周边poi - :param city: - :param location: - :return: - """ - try: - subway_df = elastic_client.scan_to_data_frame('amap_poi', { - "_source": ["name"], - "query": { - "bool": { - "must_not": [ - { - "term": { - "status": { - "value": 1 - } - } - } - ], - "filter": [ - {"terms": { - "typecode": subway_typecode - }}, - { - "term": { - "cityname": { - "value": city - } - } - }, - { - "geo_distance": { - "distance": 1000, - "location": location - } - } - ] - } - } - }); - if len(subway_df) > 0: - return subway_df['name'].tolist() - return [] - except Exception as e: - print(e) - return [] - - -def deal(city, lng, lat): +def deal(city, lng, lat, distance=1000): location = [lng, lat] - subway_df = find_around_by_location(city, location) + subway_df = poiutils.find_around_by_location(city, location, subway_typecode, distance) return subway_df - def do(): + sync1() + sync_new() + +def sync_new(distance=3000): + log.info("开始执行周边地铁配套入库") + city_dict = list(administrative_mapping_conn.find({'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] + log.info('开始查找' + city + '周边地铁') + seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) + if len(seaweed_df) <= 0: + continue + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + + actions = list() + log.info(city + '周边地铁开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, {'$set': {'subway': item['subway'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat'], 'distance': distance}}, upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + surrounding_facilities_conn.bulk_write(actions) + actions.clear() + log.info(city + '周边地铁入库完毕') + +def sync1(): log.info("开始执行周边地铁配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: diff --git a/dags/poi/business/district_supermarket.py b/dags/poi/business/district_supermarket.py index 6e167bc..17a8f4a 100644 --- a/dags/poi/business/district_supermarket.py +++ b/dags/poi/business/district_supermarket.py @@ -2,79 +2,67 @@ 小区周边1公里商场入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT import pandas as pd from pymongo import UpdateOne +import poiutils log = logging.getLogger(__name__) -elastic_client = ElasticSearchClient(host='192.168.0.206') district_facilitys_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'district_facilitys') seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'administrative_mapping') +surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', 'surrounding_facilities') # 商场code supermarket_typecode = ["060100","060101","060102"] -def find_around_by_location(city, location): - """ - 查找周边poi - :param city: - :param location: - :return: - """ - try: - supermarket_df = elastic_client.scan_to_data_frame('amap_poi', { - "_source": ["name"], - "query": { - "bool": { - "must_not": [ - { - "term": { - "status": { - "value": 1 - } - } - } - ], - "filter": [ - {"terms": { - "typecode": supermarket_typecode - }}, - { - "term": { - "cityname": { - "value": city - } - } - }, - { - "geo_distance": { - "distance": 1000, - "location": location - } - } - ] - } - } - }); - if len(supermarket_df) > 0: - return supermarket_df['name'].tolist() - return [] - except Exception as e: - print(e) - print(location) - return [] - -def deal(city, lng, lat): +def deal(city, lng, lat, distance=1000): location = [lng, lat] - supermarket_df = find_around_by_location(city, location) + supermarket_df = poiutils.find_around_by_location(city, location, supermarket_typecode, distance) return supermarket_df - def do(): + sync1() + sync_new() + +def sync_new(): + log.info("开始执行周边商场配套入库") + city_dict = list(administrative_mapping_conn.find({'type': '城市'})) + for city_item in city_dict: + city = city_item['city_short'] + log.info('开始查找' + city + '周边商场') + seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, + {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( + 10000))) + if len(seaweed_df) <= 0: + continue + seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 + seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 + seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + + actions = list() + log.info(city + '周边商场开始入库') + for index, item in seaweed_df.iterrows(): + try: + # 周边配套库中历史原因,id可能不为小区id + action = UpdateOne({'city': item['city'], 'region': item['region'], + 'district_name': item['name']}, {'$set': {'supermarket': item['supermarket'], 'city': item['city'], 'region': item['region'], + 'district_name': item['name'], 'lng': item['lng'], 'lat': item['lat']}}, upsert=True) + actions.append(action) + except Exception as e: + print(e) + if len(actions) == 5000: + district_facilitys_conn.bulk_write(actions) + actions.clear() + if len(actions) > 0: + district_facilitys_conn.bulk_write(actions) + actions.clear() + log.info(city + '周边商场入库完毕') + +def sync1(): log.info("开始执行周边商场配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: diff --git a/dags/poi/business/poiutils.py b/dags/poi/business/poiutils.py new file mode 100644 index 0000000..a770eed --- /dev/null +++ b/dags/poi/business/poiutils.py @@ -0,0 +1,54 @@ +from goojia_utils import ElasticSearchClient + +elastic_client = ElasticSearchClient(host='192.168.0.206') + +def find_around_by_location(city, location, typecode, distance=1000): + """ + 查找周边poi + :param city: + :param location: + :param distance: + :return: + """ + + try: + ret = elastic_client.scan_to_data_frame('amap_poi', { + "_source": ["name"], + "query": { + "bool": { + "must_not": [ + { + "term": { + "status": { + "value": 1 + } + } + } + ], + "filter": [ + {"terms": { + "typecode": typecode + }}, + { + "term": { + "cityname": { + "value": city + } + } + }, + { + "geo_distance": { + "distance": distance, + "location": location + } + } + ] + } + } + }); + if len(ret) > 0: + return ret['name'].tolist() + return [] + except Exception as e: + print(e) + return [] \ No newline at end of file