From b7c567b2fb5ec1d68e01a42ac2b5b0812971524c Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 29 Mar 2023 11:02:50 +0800 Subject: [PATCH] =?UTF-8?q?=E6=88=B7=E5=9D=87=E6=80=BB=E4=BB=B7=E8=A1=A5?= =?UTF-8?q?=E5=85=85=E6=96=B9=E5=BC=8F=E4=BF=AE=E6=94=B9=E5=8F=8A=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E5=88=86=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../community/total_price_complete.py | 61 ++++++++++++++++++- 1 file changed, 59 insertions(+), 2 deletions(-) diff --git a/dags/business/business/community/total_price_complete.py b/dags/business/business/community/total_price_complete.py index a37c7b4..d365136 100644 --- a/dags/business/business/community/total_price_complete.py +++ b/dags/business/business/community/total_price_complete.py @@ -6,6 +6,8 @@ import pandas as pd from goojia_utils import ElasticSearchClient, dateutils from elasticsearch import helpers import sys +from concurrent.futures import ThreadPoolExecutor +from threading import Lock log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') @@ -14,11 +16,14 @@ index_name = "community_summary" path = "/opt/airflow/dags/repo/dags/business/business/community/data" m_date = dateutils.iso_format() +mutex = Lock() def build_doc(item): doc_dict = dict() if 'total_price' in item and pd.notna(item['total_price']): doc_dict['total_price'] = int(item['total_price']) + if 'total_price_score' in item and pd.notna(item['total_price_score']): + doc_dict['credit_detail'] = {'total_price_score': item['total_price_score']} return doc_dict @@ -86,6 +91,8 @@ def complete_by_build_year_area(community_factor_df): item['city'] + "|" + item['region'] + "|2020及以后"] if item['city'] + "|" + item[ 'region'] + "|2020及以后" in build_year_area_dict else None community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] + # 按各区各建筑年代户均面积补充, 得6分 + community_factor_df.loc[community_factor_df['total_price'].notna(), 'total_price_score'] = 6 def complete_by_region_area(community_factor_df): @@ -98,18 +105,68 @@ def complete_by_region_area(community_factor_df): community_factor_df['total_price'] = community_factor_df['avg_price'] * community_factor_df['factor'] +def find_around_avg_area(city_now, region, lng, lat): + ret_es = es206_client.elastic_client.search({ + "query": { + "bool": { + "filter": [ + {"term": {"city": {"value": city_now}}}, + {"term": {"region": {"value": region}}}, + {"geo_distance": {"distance": 3000, "location": [lng, lat]}}, + {"range": {"house_avg_area": {"gt": 0}}} + ] + } + }, + "size": 0, + "aggs": { + "house_avg_area": { + "avg": { + "field": "house_avg_area" + } + } + } + }, index='community_summary') + return ret_es['aggregations']['house_avg_area']['value'] + + +def complete_by_round_area(community_factor_df): + """ + 按周边三公里小区户均面积均值补充 + """ + def deal_complete_by_round_area(index, item): + avg_price = item['avg_price'] + house_avg_area = find_around_avg_area(item['city'], item['region'], item['location']['lon'], + item['location']['lat']) + mutex.acquire(timeout=60) + community_factor_df.loc[index, 'factor'] = house_avg_area + if pd.notna(avg_price) and pd.notna(house_avg_area): + community_factor_df.loc[index, 'total_price'] = avg_price * house_avg_area + # 按周边三公里小区户均面积均值补充,得5分 + community_factor_df.loc[index, 'total_price_score'] = 5 + mutex.release() + + # for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows(): + # deal_complete_by_round_area(index, item) + with ThreadPoolExecutor(max_workers=30) as pool: + for index, item in community_factor_df[community_factor_df['total_price'].isna()].iterrows(): + pool.submit(deal_complete_by_round_area, index, item) + + + def sync(city=None): log.info(f"开始补全户均总价信息至206 {index_name}") - sql = "select city, region, name, avg_price, finish_building_date from community_summary where avg_price is not null and total_price is null " + sql = "select city, region, name, avg_price, finish_building_date, location from community_summary where avg_price is not null and total_price is null " if pd.notna(city): sql += " and city='" + city + "'" community_df = es206_client.scan_by_sql_to_data_frame(sql) if len(community_df) > 0: community_df['total_price'] = None + community_df['total_price_score'] = None complete_by_build_year_area(community_df) - complete_by_region_area(community_df) + #complete_by_region_area(community_df) + complete_by_round_area(community_df) save(community_df) log.info(f"完成补全户均总价信息至206 {index_name}")