From e69058d28e599a6f080166de07a6253a17daa6ce Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Thu, 3 Nov 2022 15:49:27 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=87=8D=E7=82=B9=E5=B0=8F?= =?UTF-8?q?=E5=AD=A6=E3=80=81=E9=87=8D=E7=82=B9=E4=B8=AD=E5=AD=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/business/community/education2.py | 104 ++++++++++++++++++ 1 file changed, 104 insertions(+) create mode 100644 dags/business/business/community/education2.py diff --git a/dags/business/business/community/education2.py b/dags/business/business/community/education2.py new file mode 100644 index 0000000..789a4ab --- /dev/null +++ b/dags/business/business/community/education2.py @@ -0,0 +1,104 @@ +""" +教育资源相关信息同步至ES206 +""" +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() + +count = 0 +mutex = Lock() + + +def deal(actions, item): + global count + curr_city = item['city'] + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + + if pd.isna(lng) or pd.isna(lat): + return + + doc_info = dict() + + # 小学 + primary_school = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "学校", "小学") + + key_primary_school = primary_school[primary_school['secondary_type'] == '重点'] + ordinary_primary_school = primary_school[primary_school['secondary_type'] != '重点'] + + doc_info['key_primary_school'] = len(key_primary_school) + if len(key_primary_school) > 0: + doc_info['key_primary_school'] = ",".join(key_primary_school) + + doc_info['ordinary_primary_school'] = len(ordinary_primary_school) + if len(ordinary_primary_school) > 0: + doc_info['ordinary_primary_school'] = ",".join(ordinary_primary_school) + + # 中学 + middle_school = poiutils.find_around_by_location(curr_city, 2000, [lng, lat], "学校", "中学") + + key_middle_school = middle_school[middle_school['secondary_type'] == '重点'] + ordinary_middle_school = middle_school[middle_school['secondary_type'] != '重点'] + + doc_info['key_middle_school'] = len(key_middle_school) + if len(key_middle_school) > 0: + doc_info['key_middle_school'] = ",".join(key_middle_school) + + doc_info['ordinary_middle_school'] = len(ordinary_middle_school) + if len(ordinary_middle_school) > 0: + doc_info['ordinary_middle_school'] = ",".join(ordinary_middle_school) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + mutex.acquire(timeout=60) + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + except Exception as e: + log.error(e) + actions.clear() + mutex.release() + + +def sync(city=None): + log.info(f"开始同步教育资源【小学、中学】至206 {index_name}") + seaweed_query = {'status': 0, 'cat': 'district'} + if pd.notna(city): + seaweed_query['city'] = city + + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + + actions = list() + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(deal, actions, item) + + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + log.error(e) + actions.clear() + + log.info(f"完成同步教育资源至206 {index_name}") + + +sync() \ No newline at end of file