diff --git a/dags/business/business/community/total_price_complete.py b/dags/business/business/community/total_price_complete.py index a37c7b4..d365136 100644 --- a/dags/business/business/community/total_price_complete.py +++ b/dags/business/business/community/total_price_complete.py @@ -6,6 +6,8 @@ import pandas as pd from goojia_utils import ElasticSearchClient, dateutils from elasticsearch import helpers import sys +from concurrent.futures import ThreadPoolExecutor +from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -14,11 +16,14 @@ index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" m_date = dateutils.iso_format() +mutex = Lock() def build_doc(item): doc_dict = dict() if 'total_price' in item and pd.notna(item['total_price']): doc_dict['total_price'] = int(item['total_price']) + if 'total_price_score' in item and pd.notna(item['total_price_score']): + doc_dict['credit_detail'] = {'total_price_score': item['total_price_score']} return doc_dict @@ -86,6 +91,8 @@ def complete_by_build_year_area(community_factor_df): item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item[ 'region'] + "|2020及以后" in build_year_area_dict else None community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] + # 按各区各建筑年代户均面积补充, 得6分 + community_factor_df.loc[community_factor_df['total_price'].notna(), 'total_price_score'] = 6 def complete_by_region_area(community_factor_df): @@ -98,18 +105,68 @@ def complete_by_region_area(community_factor_df): community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] +def find_around_avg_area(city_now, region, lng, lat): + ret_es = es206_client.elastic_client.search({ + "query": { + "bool": { + "filter": [ + {"term": {"city": {"value": city_now}}}, + {"term": {"region": {"value": region}}}, + {"geo_distance": {"distance": 3000, "location": [lng, lat]}}, + {"range": {"house_avg_area": {"gt": 0}}} + ] + } + }, + "size": 0, + "aggs": { + "house_avg_area": { + "avg": { + "field": "house_avg_area" + } + } + } + }, index='community_summary') + return ret_es['aggregations']['house_avg_area']['value'] + + +def complete_by_round_area(community_factor_df): + """ + 按周边三公里小区户均面积均值补充 + """ + def deal_complete_by_round_area(index, item): + avg_price = item['avg_price'] + house_avg_area = find_around_avg_area(item['city'], item['region'], item['location']['lon'], + item['location']['lat']) + mutex.acquire(timeout=60) + community_factor_df.loc[index, 'factor'] = house_avg_area + if pd.notna(avg_price) and pd.notna(house_avg_area): + community_factor_df.loc[index, 'total_price'] = avg_price * house_avg_area + # 按周边三公里小区户均面积均值补充,得5分 + community_factor_df.loc[index, 'total_price_score'] = 5 + mutex.release() + + # for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows(): + # deal_complete_by_round_area(index, item) + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows(): + pool.submit(deal_complete_by_round_area, index, item) + + + def sync(city=None): log.info(f"开始补全户均总价信息至206 {index_name}") - sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null " + sql = "select city, region, name, avg_price, finish_building_date, location from community_summary where avg_price is not null and total_price is null " if pd.notna(city): sql += " and city='" + city + "'" community_df = es206_client.scan_by_sql_to_data_frame(sql) if len(community_df) > 0: community_df['total_price'] = None + community_df['total_price_score'] = None complete_by_build_year_area(community_df) - complete_by_region_area(community_df) + #complete_by_region_area(community_df) + complete_by_round_area(community_df) save(community_df) log.info(f"完成补全户均总价信息至206 {index_name}")