From 83cdba3406f346f2bb8457b7793b506b6804b732 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 9 May 2023 11:44:45 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=A0=BC=E5=AD=90=E5=9D=87?= =?UTF-8?q?=E4=BB=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/grid_price.py | 41 ++++++++++++++++++++++++---- 1 file changed, 36 insertions(+), 5 deletions(-) diff --git a/dags/business/business/grid_price.py b/dags/business/business/grid_price.py index 88fa42d..3e73225 100644 --- a/dags/business/business/grid_price.py +++ b/dags/business/business/grid_price.py @@ -2,6 +2,7 @@ import logging import sys from goojia_utils import ElasticSearchClient, commutils import pandas as pd +from elasticsearch import helpers import time log = logging.getLogger(__name__) @@ -42,7 +43,7 @@ def find_comm_by_grid(city, geo_shape): estate_type_list = ["普通住宅", "别墅", "商住"] def sync(city): - sql = f"select * from grid " + sql = f"select * from test_grid " if pd.notna(city): sql += f" where city='{city}' " origin_grid_df = es206_client.scan_by_sql_to_data_frame(sql) @@ -55,10 +56,15 @@ def sync(city): avg_price_df = avg_price_df.rename(columns={'district_name': 'name'}) # 求解格子均价 actions = list() + count = 0 + log.info(f"开始计算{curr_city}格子价格") for index, item in grid_df.iterrows(): comm_info_df = find_comm_by_grid(item['city'], item['fence']['coordinates'][0]) - action = dict() - ret = dict() + if len(comm_info_df) <= 0: + continue + grid_info = dict() + price_list = list() + price_info = dict() for estate_type in estate_type_list: # 计算不同物业类型的小区均价 comm_info_estate_type_df = comm_info_df[comm_info_df['estate_type'] == estate_type].copy() @@ -66,6 +72,31 @@ def sync(city): comm_info_estate_type_df = pd.merge(comm_info_estate_type_df, avg_price_df, how='left', on=['city', 'region', 'name']) comm_info_estate_type_df = comm_info_estate_type_df[comm_info_estate_type_df['avg_price'].notna()] if len(comm_info_estate_type_df) > 0: - ret[estate_type] = int(comm_info_estate_type_df['avg_price'].mean()) - action['district_price'] = {''} + price_info['type'] = 'district' + price_info['secondly_type'] = estate_type + price_info['price'] = int(comm_info_estate_type_df['avg_price'].mean()) + price_list.append(price_info) + if len(price_list) > 0: + grid_info['price'] = price_list + action = { + "_op_type": "update", + "_index": "test_grid", + "_id": item['_id'], + "doc": grid_info, + "_retry_on_conflict": 5 + } + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + actions.clear() + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + actions.clear() + log.info(f"完成计算{curr_city}格子价格") + + sync(city_name)