From 519fffccb9c487539e5de6bdb500bf84cee3f03c Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Fri, 21 Oct 2022 17:57:30 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=8C=BB=E7=96=97=E8=B5=84?= =?UTF-8?q?=E6=BA=90=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/community_summary.py | 4 +- dags/business/business/community/medical.py | 86 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 dags/business/business/community/medical.py diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index a4bc10a..d06be2f 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -7,6 +7,7 @@ import trade import avg_cost import total_price import traffic +import medical log = logging.getLogger(__name__) @@ -16,9 +17,10 @@ log.info("同步类型:" + type) if type == 'all': #comm.sync() - price.sync() + #price.sync() #sell.sync() #avg_cost.sync() #trade.sync() #total_price.sync() #traffic.sync() + medical.sync() diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py new file mode 100644 index 0000000..d37d391 --- /dev/null +++ b/dags/business/business/community/medical.py @@ -0,0 +1,86 @@ +""" +医疗资源相关信息同步至ES206 +""" +import logging +import pandas as pd +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +log = logging.getLogger(__name__) +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + +count = 0 +mutex = Lock() + + +def deal(actions, item): + global count + curr_city = item['city'] + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + + if pd.isna(lng) or pd.isna(lat): + return + + doc_info = dict() + # 三甲 + top_three_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "三甲医院") + doc_info['top_three_hospital_num'] = len(top_three_hospital) + doc_info['top_three_hospital'] = ",".join(top_three_hospital) + + # 非三甲 + non_3a_hospital = poiutils.find_around_by_location(curr_city, 5000, [lng, lat], "医院", "普通医院") + doc_info['non_3a_hospital_num'] = len(non_3a_hospital) + doc_info['non_3a_hospital'] = ",".join(non_3a_hospital) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + mutex.acquire(timeout=60) + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + except Exception as e: + log.error(e) + actions.clear() + mutex.release() + + +def sync(city=None): + log.info(f"开始同步医院至206 {index_name}") + seaweed_query = {'status': 0, 'cat': 'district'} + if pd.notna(city): + seaweed_query['city'] = city + + seaweed_df = pd.DataFrame( + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}) + .batch_size(100000))) + + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + actions = list() + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(deal, actions, item) + + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + print(e) + actions.clear() + + log.info(f"完成同步医院至206 {index_name}") \ No newline at end of file