diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 318837e..196a3a2 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -6,6 +6,7 @@ import sell import trade import avg_cost import total_price +import traffic log = logging.getLogger(__name__) @@ -19,4 +20,5 @@ if type == 'all': #sell.sync() #avg_cost.sync() #trade.sync() - total_price.sync() \ No newline at end of file + #total_price.sync() + traffic.sync() \ No newline at end of file diff --git a/dags/business/business/community/facilities.py b/dags/business/business/community/facilities.py deleted file mode 100644 index fb71fbd..0000000 --- a/dags/business/business/community/facilities.py +++ /dev/null @@ -1,73 +0,0 @@ -""" -周边配套相关信息同步至ES206 -""" -import logging -import pandas as pd -from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils -from elasticsearch import helpers - -log = logging.getLogger(__name__) -seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') -es206_client = ElasticSearchClient(host='192.168.0.206') - -index_name = "community_summary" -m_date = dateutils.iso_format() - - -def build_doc(item): - doc_dict = dict() - if pd.notna(item['region_avg_price']): - doc_dict['region_avg_price'] = item['region_avg_price'] - return doc_dict - - -def sync(city=None): - log.info(f"开始同步周边配套至206 {index_name}") - seaweed_query = {'cat': 'district', 'status': 0} - if pd.notna(city): - seaweed_query['city'] = city - cities = [city] - else: - cities = None - seaweed_df = pd.DataFrame( - list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1}) - .batch_size(100000))) - - # 小区所在区域均价 - region_price_df = commutils.get_region_price(city=city) - region_price_df = region_price_df[['city', 'region', 'avg_price']] - region_price_df = region_price_df.rename(columns={'avg_price': 'region_avg_price'}) - seaweed_df = pd.merge(seaweed_df, region_price_df, how='left', on=['city', 'region']) - - - - seaweed_df['_id'] = seaweed_df['_id'].astype(str) - - actions = list() - count = 0 - for index, item in seaweed_df.iterrows(): - doc_info = build_doc(item) - - action = { - "_op_type": "update", - "_index": index_name, - "_id": item['_id'], - "doc": doc_info - } - - actions.append(action) - if len(actions) == 5000: - try: - helpers.bulk(es206_client.elastic_client, actions) - count += len(actions) - print("更新:" + str(count)) - except Exception as e: - print(e) - actions.clear() - if len(actions) > 0: - try: - helpers.bulk(es206_client.elastic_client, actions) - except Exception as e: - print(e) - actions.clear() - log.info(f"完成同步周边配套至206 {index_name}") \ No newline at end of file diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py new file mode 100644 index 0000000..576ca11 --- /dev/null +++ b/dags/business/business/community/traffic.py @@ -0,0 +1,96 @@ +""" +交通相关信息同步至ES206 +""" +import logging +import pandas as pd +from goojia_utils import DEV_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils, poiutils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +from threading import Lock + +log = logging.getLogger(__name__) +seaweed_conn = DEV_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + +count = 0 +mutex = Lock() + + +def deal(actions, item): + global count + curr_city = item['city'] + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + + if pd.isna(lng) or pd.isna(lat): + return + + doc_info = dict() + # 公交 + bus_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "公交") + doc_info['bus_station_num'] = len(bus_station) + doc_info['bus_station'] = ",".join(bus_station) + + bus_doc = poiutils.find_nearest_by_location(curr_city, 1000, [lng, lat], "交通", "公交") + if pd.notna(bus_doc): + doc_info['nearest_bus_station_distance'] = int(bus_doc['sort'][0]) + doc_info['nearest_bus_station'] = bus_doc['_source']['name'] + + # 地铁 + subway_station = poiutils.find_around_by_location(curr_city, 1000, [lng, lat], "交通", "地铁") + doc_info['subway_station_num'] = len(subway_station) + doc_info['subway_station'] = ",".join(subway_station) + + bus_doc = poiutils.find_nearest_by_location(curr_city, 1000, [lng, lat], "交通", "地铁") + if pd.notna(bus_doc): + doc_info['nearest_metro_station_distance'] = int(bus_doc['sort'][0]) + doc_info['nearest_metro_station'] = bus_doc['_source']['name'] + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + mutex.acquire(timeout=10) + actions.append(action) + if len(actions) == 1000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) + except Exception as e: + print(e) + actions.clear() + mutex.release() + + +def sync(city=None): + log.info(f"开始同步周边配套至206 {index_name}") + seaweed_query = {'status': 0, 'cat': 'district'} + if pd.notna(city): + seaweed_query['city'] = city + + seaweed_df = pd.DataFrame( + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1}) + .batch_size(100000))) + + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + actions = list() + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(deal, actions, item) + + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + print(e) + actions.clear() + + log.info(f"完成同步周边配套至206 {index_name}") \ No newline at end of file