From 13bb8577014ad5f83d2639c5d410357db18fd726 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 9 May 2023 13:17:24 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=A0=BC=E5=AD=90=E5=9D=87?= =?UTF-8?q?=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/grid_price.py | 70 +++++++++++++++++----------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/dags/business/business/grid_price.py b/dags/business/business/grid_price.py index 3e73225..1f6d98b 100644 --- a/dags/business/business/grid_price.py +++ b/dags/business/business/grid_price.py @@ -3,7 +3,7 @@ import sys from goojia_utils import ElasticSearchClient, commutils import pandas as pd from elasticsearch import helpers -import time +import transbigdata as tbd log = logging.getLogger(__name__) @@ -12,33 +12,39 @@ if pd.notna(city_name): log.info("同步城市:" + city_name) es206_client = ElasticSearchClient(host='192.168.0.206') +# 中国地理位置边界[lon1,lat1,lon2,lat2] (WGS84坐标系) 其中,lon1,lat1是左下角坐标,lon2,lat2是右上角坐标, +china_bounds = [73.66, 3.86, 135.05, 53.55] +# 栅格大小(米) +accuracy = 1000 +# 生成栅格参数 +params_rect = tbd.area_to_params(china_bounds, accuracy=accuracy) -def find_comm_by_grid(city, geo_shape): - comm_info_df = es206_client.scan_to_data_frame('community_summary', { - "_source": ["id", "city", "region", "name", "estate_type"], - "query": { - "bool": { - "filter": [ - { - "term": { - "city": { - "value": city - } - } - }, - { - "geo_polygon": { - "location": { - "points": geo_shape - } - } - } - ] - } - } - }) - return comm_info_df +# def find_comm_by_grid(city, geo_shape): +# comm_info_df = es206_client.scan_to_data_frame('community_summary', { +# "_source": ["id", "city", "region", "name", "estate_type"], +# "query": { +# "bool": { +# "filter": [ +# { +# "term": { +# "city": { +# "value": city +# } +# } +# }, +# { +# "geo_polygon": { +# "location": { +# "points": geo_shape +# } +# } +# } +# ] +# } +# } +# }) +# return comm_info_df estate_type_list = ["普通住宅", "别墅", "商住"] @@ -54,12 +60,22 @@ def sync(city): avg_price_df = commutils.get_avg_price(curr_city) avg_price_df = avg_price_df[['city', 'region', 'district_name', 'avg_price']] avg_price_df = avg_price_df.rename(columns={'district_name': 'name'}) + + comm_df = es206_client.scan_by_sql_to_data_frame(f"select id, city, region, name, estate_type, location from community_summary where city='{city}' ") + + def deal_lng_lat(x): + location = x['location'] + if pd.notna(location): + lon_lat_col = tbd.GPS_to_grid(location['lon'], location['lat'], params_rect) + x['grid_id'] = str(lon_lat_col[0]) + "_" + str(lon_lat_col[1]) + return x + comm_df = comm_df.apply(lambda x: deal_lng_lat(x), axis=1) # 求解格子均价 actions = list() count = 0 log.info(f"开始计算{curr_city}格子价格") for index, item in grid_df.iterrows(): - comm_info_df = find_comm_by_grid(item['city'], item['fence']['coordinates'][0]) + comm_info_df = comm_df[comm_df['grid_id'] == item['_id']] if len(comm_info_df) <= 0: continue grid_info = dict()