From 67eed5a9dfbcbe7faf35c8ff7503403aab6e4615 Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Wed, 20 Nov 2024 18:59:55 +0800 Subject: [PATCH] =?UTF-8?q?=E5=91=A8=E8=BE=B9POI=E6=94=B9=E4=B8=BA?= =?UTF-8?q?=E5=A4=9A=E7=BA=BF=E7=A8=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/poi/business/district_bank.py | 29 +++++++++++++---------- dags/poi/business/district_bus.py | 28 ++++++++++++---------- dags/poi/business/district_hospital.py | 26 ++++++++++++-------- dags/poi/business/district_school.py | 29 +++++++++++++---------- dags/poi/business/district_subway.py | 29 +++++++++++++---------- dags/poi/business/district_supermarket.py | 29 +++++++++++++---------- 6 files changed, 97 insertions(+), 73 deletions(-) diff --git a/dags/poi/business/district_bank.py b/dags/poi/business/district_bank.py index 64d20db..2cc6d9e 100644 --- a/dags/poi/business/district_bank.py +++ b/dags/poi/business/district_bank.py @@ -2,7 +2,7 @@ 小区周边1公里银行入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT +from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread import pandas as pd from pymongo import UpdateOne @@ -26,10 +26,15 @@ bank_typecode = ["160100", "160101", "160102", "160103", "160104", "160105", "16 "160144", "160145", "160146", "160147", "160148", "160149", "160150", "160151", "160152", "160200"] +def deal3(item): + location = [item['lng'], item['lat']] + item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 3000) + return item -def deal(city, lng, lat, distance=1000): - location = [lng, lat] - return poiutils.find_around_by_location(city, location, bank_typecode, distance) +def deal(item): + location = [item['lng'], item['lat']] + item['bank'] = poiutils.find_around_by_location(item['city_display'], location, bank_typecode, 1000) + return item def do(): sync_new() @@ -39,11 +44,10 @@ def sync_new(distance=3000): """ 周边3公里POI入库 """ - log.info("开始执行周边银行配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边银行') + city_display = city_item['city'] seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) @@ -51,10 +55,11 @@ def sync_new(distance=3000): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + seaweed_df['city_display'] = city_display + seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30) + #seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) actions = list() - log.info(city + '周边银行开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id @@ -75,11 +80,10 @@ def sync_new(distance=3000): log.info(city + '周边银行入库完毕') def sync1(): - log.info("开始执行周边银行配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边银行') + city_display = city_item['city'] seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) @@ -87,10 +91,11 @@ def sync1(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + seaweed_df['city_display'] = city_display + seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30) + #seaweed_df['bank'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() - log.info(city + '周边银行开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id diff --git a/dags/poi/business/district_bus.py b/dags/poi/business/district_bus.py index 29237d9..20dfef1 100644 --- a/dags/poi/business/district_bus.py +++ b/dags/poi/business/district_bus.py @@ -2,7 +2,7 @@ 小区周边1公里交通地铁入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT +from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread import pandas as pd from pymongo import UpdateOne import poiutils @@ -18,9 +18,15 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' bus_typecode = ["150700"] -def deal(city, lng, lat, distance=1000): - location = [lng, lat] - return poiutils.find_around_by_location(city, location, bus_typecode, distance) +def deal3(item): + location = [item['lng'], item['lat']] + item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 3000) + return item + +def deal(item): + location = [item['lng'], item['lat']] + item['bus'] = poiutils.find_around_by_location(item['city_display'], location, bus_typecode, 1000) + return item def do(): sync_new() @@ -28,11 +34,10 @@ def do(): def sync_new(distance=3000): - log.info("开始执行周边交通配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边交通') + city_display = city_item['city'] seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( @@ -41,10 +46,10 @@ def sync_new(distance=3000): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30) + #seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) actions = list() - log.info(city + '周边交通开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id @@ -65,11 +70,10 @@ def sync_new(distance=3000): log.info(city + '周边交通入库完毕') def sync1(): - log.info("开始执行周边交通配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边交通') + city_display = city_item['city'] seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( @@ -78,10 +82,10 @@ def sync1(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30) + #seaweed_df['bus'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() - log.info(city + '周边交通开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id diff --git a/dags/poi/business/district_hospital.py b/dags/poi/business/district_hospital.py index c7b505d..5346de6 100644 --- a/dags/poi/business/district_hospital.py +++ b/dags/poi/business/district_hospital.py @@ -2,7 +2,7 @@ 小区周边1公里医院入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT +from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread import poiutils import pandas as pd @@ -20,10 +20,15 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' hospital_typecode = ["090100", "090101", "090200"] -def deal(city, lng, lat, distance=1000): - location = [lng, lat] - hospital_df = poiutils.find_around_by_location(city, location, hospital_typecode, distance) - return hospital_df +def deal3(item): + location = [item['lng'], item['lat']] + item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 3000) + return item + +def deal(item): + location = [item['lng'], item['lat']] + item['hospital'] = poiutils.find_around_by_location(item['city_display'], location, hospital_typecode, 1000) + return item def do(): @@ -35,6 +40,7 @@ def sync_new(distance=3000): city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] + city_display = city_item['city'] log.info('开始查找' + city + '周边医院') seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, @@ -43,7 +49,8 @@ def sync_new(distance=3000): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30) + #seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) actions = list() log.info(city + '周边医院开始入库') @@ -69,11 +76,10 @@ def sync_new(distance=3000): def sync1(): - log.info("开始执行周边医院配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边医院') + city_display = city_item['city'] seaweed_df = pd.DataFrame( list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) @@ -81,10 +87,10 @@ def sync1(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30) + #seaweed_df['hospital'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() - log.info(city + '周边医院开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id diff --git a/dags/poi/business/district_school.py b/dags/poi/business/district_school.py index bbf3184..d9ffde1 100644 --- a/dags/poi/business/district_school.py +++ b/dags/poi/business/district_school.py @@ -2,7 +2,7 @@ 小区周边1公里教育入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT +from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread import pandas as pd from pymongo import UpdateOne import poiutils @@ -18,10 +18,15 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' school_typecode = ["141200", "141201", "141202", "141203", "141204"] -def deal(city, lng, lat, distance=1000): - location = [lng, lat] - school_df = poiutils.find_around_by_location(city, location, school_typecode, distance) - return school_df +def deal3(item): + location = [item['lng'], item['lat']] + item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 3000) + return item + +def deal(item): + location = [item['lng'], item['lat']] + item['school'] = poiutils.find_around_by_location(item['city_display'], location, school_typecode, 1000) + return item def do(): @@ -29,21 +34,20 @@ def do(): sync1() def sync_new(distance=3000): - log.info("开始执行周边教育配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边教育') + city_display = city_item['city'] seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) if len(seaweed_df) <= 0: continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30) + #seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) actions = list() - log.info(city + '周边教育开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id @@ -62,21 +66,20 @@ def sync_new(distance=3000): log.info(city + '周边教育入库完毕') def sync1(): - log.info("开始执行周边教育配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边教育') + city_display = city_item['city'] seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size(10000))) if len(seaweed_df) <= 0: continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30) + #seaweed_df['school'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() - log.info(city + '周边教育开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id diff --git a/dags/poi/business/district_subway.py b/dags/poi/business/district_subway.py index a363826..cc9fbf1 100644 --- a/dags/poi/business/district_subway.py +++ b/dags/poi/business/district_subway.py @@ -2,7 +2,7 @@ 小区周边1公里地铁地铁入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, process_dataframe_multithread import pandas as pd from pymongo import UpdateOne import poiutils @@ -18,21 +18,25 @@ administrative_mapping_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' # 地铁code subway_typecode = ["150500"] -def deal(city, lng, lat, distance=1000): - location = [lng, lat] - subway_df = poiutils.find_around_by_location(city, location, subway_typecode, distance) - return subway_df +def deal3(item): + location = [item['lng'], item['lat']] + item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 3000) + return item + +def deal(item): + location = [item['lng'], item['lat']] + item['subway'] = poiutils.find_around_by_location(item['city_display'], location, subway_typecode, 1000) + return item def do(): sync_new() sync1() def sync_new(distance=3000): - log.info("开始执行周边地铁配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边地铁') + city_display = city_item['city'] seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( 10000))) @@ -40,10 +44,10 @@ def sync_new(distance=3000): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30) + #seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat'], distance), axis=1) actions = list() - log.info(city + '周边地铁开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id @@ -62,11 +66,10 @@ def sync_new(distance=3000): log.info(city + '周边地铁入库完毕') def sync1(): - log.info("开始执行周边地铁配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边地铁') + city_display = city_item['city'] seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( 10000))) @@ -74,10 +77,10 @@ def sync1(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30) + #seaweed_df['subway'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() - log.info(city + '周边地铁开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id diff --git a/dags/poi/business/district_supermarket.py b/dags/poi/business/district_supermarket.py index ca03698..947d71b 100644 --- a/dags/poi/business/district_supermarket.py +++ b/dags/poi/business/district_supermarket.py @@ -2,7 +2,7 @@ 小区周边1公里商场入库业务逻辑 """ import logging -from goojia_utils import MAIN_MONGO_CLIENT +from goojia_utils import MAIN_MONGO_CLIENT, process_dataframe_multithread import pandas as pd from pymongo import UpdateOne import poiutils @@ -19,21 +19,25 @@ surrounding_facilities_conn = MAIN_MONGO_CLIENT.get_collection('fangjia_base', ' supermarket_typecode = ["060100","060101","060102"] -def deal(city, lng, lat, distance=1000): - location = [lng, lat] - supermarket_df = poiutils.find_around_by_location(city, location, supermarket_typecode, distance) - return supermarket_df +def deal3(item): + location = [item['lng'], item['lat']] + item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 3000) + return item + +def deal(item): + location = [item['lng'], item['lat']] + item['supermarket'] = poiutils.find_around_by_location(item['city_display'], location, supermarket_typecode, 1000) + return item def do(): sync_new() sync1() def sync_new(): - log.info("开始执行周边商场配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边商场') + city_display = city_item['city'] seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( 10000))) @@ -41,10 +45,10 @@ def sync_new(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal3, 30) + #seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() - log.info(city + '周边商场开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id @@ -63,11 +67,10 @@ def sync_new(): log.info(city + '周边商场入库完毕') def sync1(): - log.info("开始执行周边商场配套入库") city_dict = list(administrative_mapping_conn.find({'type': '城市'})) for city_item in city_dict: city = city_item['city_short'] - log.info('开始查找' + city + '周边商场') + city_display = city_item['city'] seaweed_df = pd.DataFrame(list(seaweed_conn.find({'city': city, 'cat': 'district', 'status': 0, 'lng': {'$ne': None}}, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}).batch_size( 10000))) @@ -75,10 +78,10 @@ def sync1(): continue seaweed_df['lng'] = seaweed_df['lng'] / 10 ** 10 seaweed_df['lat'] = seaweed_df['lat'] / 10 ** 10 - seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) + seaweed_df = process_dataframe_multithread(seaweed_df, deal, 30) + #seaweed_df['supermarket'] = seaweed_df.apply(lambda x: deal(city_item['city'], x['lng'], x['lat']), axis=1) actions = list() - log.info(city + '周边商场开始入库') for index, item in seaweed_df.iterrows(): try: # 周边配套库中历史原因,id可能不为小区id