diff --git a/dags/business/business/community/business_district.py b/dags/business/business/community/business_district.py new file mode 100644 index 0000000..17be6a1 --- /dev/null +++ b/dags/business/business/community/business_district.py @@ -0,0 +1,114 @@ +""" +小区最近商圈信息同步至ES206 +""" +import logging +import pandas as pd +from goojia_utils import ElasticSearchClient, commutils, dateutils, poiutils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() + +count = 0 +mutex = Lock() + + +def find_nearest_by_location(city, distance, location): + try: + scan = es206_client.scan('business_district', { + "size": 1, + "query": { + "bool": { + "filter": [ + { + "term": { + "city": city + } + }, + { + "geo_distance": { + "distance": distance, + "location": location + } + } + ] + } + }, + "sort": [ + { + "_geo_distance": { + "location": location, + "order": "asc" + } + } + ] + }, preserve_order=True) + for doc in scan: + return doc + return None + except Exception as e: + logging.error(e) + return None + + +def deal(actions, item): + global count + curr_city = item['city'] + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + + if pd.isna(lng) or pd.isna(lat): + return + + doc_info = dict() + # 商圈 + business_district_doc = find_nearest_by_location(curr_city, 10000, [lng, lat]) + if pd.notna(business_district_doc): + doc_info['nearest_business_district_distance'] = int(business_district_doc['sort'][0]) + doc_info['nearest_business_district'] = business_district_doc['_source']['name'] + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + mutex.acquire(timeout=60) + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + actions.clear() + mutex.release() + + +def sync(city=None): + log.info(f"开始同步商圈至206 {index_name}") + seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index() + seaweed_df = seaweed_df.drop(columns=['index']) + + actions = list() + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(deal, actions, item) + + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + + log.info(f"完成同步商圈至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 47bdab3..7e4c64a 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -76,11 +76,11 @@ def deal(actions, item): def sync(city=None): log.info(f"开始同步生活配套至206 {index_name}") - seaweed_query = {'status': 0, 'cat': 'district'} - if pd.notna(city): - seaweed_query['city'] = city - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') + if pd.notna(city): + seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() + seaweed_df = seaweed_df.reset_index() + seaweed_df = seaweed_df.drop(columns=['index']) actions = list() with ThreadPoolExecutor(max_workers=30) as pool: