From 25f5cb61583a1998703e4b951084d31e9b67bef1 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 24 Oct 2022 11:21:49 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=94=9F=E6=B4=BB=E9=85=8D?= =?UTF-8?q?=E5=A5=97=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/life.py | 99 +++++++++++++++++++++ dags/business/business/community/medical.py | 8 +- dags/business/business/community/traffic.py | 6 +- 3 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 dags/business/business/community/life.py diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py new file mode 100644 index 0000000..6a81765 --- /dev/null +++ b/dags/business/business/community/life.py @@ -0,0 +1,99 @@ +""" +生活配套相关信息同步至ES206 +""" +import logging +import pandas as pd +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +log = logging.getLogger(__name__) +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + +count = 0 +mutex = Lock() + + +def deal(actions, item): + global count + curr_city = item['city'] + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + + if pd.isna(lng) or pd.isna(lat): + return + + doc_info = dict() + + # 商场 + mall = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "生活配套", "商场") + doc_info['mall_num'] = len(mall) + if len(mall) > 0: + doc_info['mall'] = ",".join(mall) + + # 超市 + supermarket = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "生活配套", "超市") + doc_info['supermarket_num'] = len(supermarket) + if len(supermarket) > 0: + doc_info['supermarket'] = ",".join(supermarket) + + # 公园 + # non_3a_hospital = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "医院", "普通医院") + # doc_info['non_3a_hospital_num'] = len(non_3a_hospital) + # doc_info['non_3a_hospital'] = ",".join(non_3a_hospital) + # + # # 餐厅 + # non_3a_hospital = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "医院", "普通医院") + # doc_info['non_3a_hospital_num'] = len(non_3a_hospital) + # doc_info['non_3a_hospital'] = ",".join(non_3a_hospital) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + mutex.acquire(timeout=60) + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + except Exception as e: + log.error(e) + actions.clear() + mutex.release() + + +def sync(city=None): + log.info(f"开始同步生活配套至206 {index_name}") + seaweed_query = {'status': 0, 'cat': 'district'} + if pd.notna(city): + seaweed_query['city'] = city + + seaweed_df = pd.DataFrame( + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}) + .batch_size(100000))) + + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + actions = list() + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(deal, actions, item) + + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + log.error(e) + actions.clear() + + log.info(f"完成同步生活配套至206 {index_name}") \ No newline at end of file diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index d37d391..540135c 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -32,12 +32,14 @@ def deal(actions, item): # 三甲 top_three_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "三甲医院") doc_info['top_three_hospital_num'] = len(top_three_hospital) - doc_info['top_three_hospital'] = ",".join(top_three_hospital) + if len(top_three_hospital) > 0: + doc_info['top_three_hospital'] = ",".join(top_three_hospital) # 非三甲 non_3a_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "普通医院") doc_info['non_3a_hospital_num'] = len(non_3a_hospital) - doc_info['non_3a_hospital'] = ",".join(non_3a_hospital) + if len(non_3a_hospital) > 0: + doc_info['non_3a_hospital'] = ",".join(non_3a_hospital) action = { "_op_type": "update", @@ -80,7 +82,7 @@ def sync(city=None): try: helpers.bulk(es206_client.elastic_client, actions) except Exception as e: - print(e) + log.error(e) actions.clear() log.info(f"完成同步医院至206 {index_name}") \ No newline at end of file diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 761569c..a561009 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -32,7 +32,8 @@ def deal(actions, item): # 公交 bus_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "公交") doc_info['bus_station_num'] = len(bus_station) - doc_info['bus_station'] = ",".join(bus_station) + if len(bus_station) > 0: + doc_info['bus_station'] = ",".join(bus_station) bus_doc = poiutils.find_nearest_by_location(curr_city, 1000, [lng, lat], "交通", "公交") if pd.notna(bus_doc): @@ -42,7 +43,8 @@ def deal(actions, item): # 地铁 subway_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "地铁") doc_info['subway_station_num'] = len(subway_station) - doc_info['subway_station'] = ",".join(subway_station) + if len(subway_station) > 0: + doc_info['subway_station'] = ",".join(subway_station) bus_doc = poiutils.find_nearest_by_location(curr_city, 1000, [lng, lat], "交通", "地铁") if pd.notna(bus_doc):