From 2451f5099ff540781ba3867aa6b39b6d4ea04358 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 18 Oct 2022 18:09:05 +0800 Subject: [PATCH] =?UTF-8?q?=E6=8B=86=E5=BC=80=E6=8C=82=E7=89=8C=E4=B8=8E?= =?UTF-8?q?=E6=88=B7=E5=9D=87(=E7=89=B9=E6=85=A2)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/sell.py | 13 --- .../business/community/total_price.py | 97 +++++++++++++++++++ 2 files changed, 97 insertions(+), 13 deletions(-) create mode 100644 dags/business/business/community/total_price.py diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index bb744db..3729f5a 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -19,8 +19,6 @@ def build_doc(item): doc_dict = dict() if 'three_months_sell_num' in item and pd.notna(item['three_months_sell_num']): doc_dict['three_months_sell_num'] = int(item['three_months_sell_num']) - if 'total_price' in item and pd.notna(item['total_price']): - doc_dict['total_price'] = int(item['total_price']) return doc_dict @@ -33,9 +31,6 @@ def sync(city=None): seaweed_df = pd.DataFrame( list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) - # 加载库中均价数据 - community_summary_df = es206_client.scan_by_sql_to_data_frame("SELECT city, region, name, avg_price FROM community_summary") - with ThreadPoolExecutor(50) as pool: def deal(index, item): if index % 1000 == 0: @@ -54,14 +49,6 @@ def sync(city=None): sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93) seaweed_df.loc[index, 'three_months_sell_num'] = sample_num - # 户均总价 - avg_price = community_summary_df.loc[(community_summary_df['city'] == curr_city) & (community_summary_df['region'] == curr_region) & (community_summary_df['name'] == curr_name)].reset_index().loc[0, 'avg_price'] - if pd.notna(avg_price): - min_avg_price = 0.8 * avg_price - max_avg_rice = 1.2 * avg_price - total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=93, min_avg_price=min_avg_price, max_avg_price=max_avg_rice) - seaweed_df.loc[index, 'total_price'] = total_price - for index, item in seaweed_df.iterrows(): pool.submit(deal, index, item) diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py new file mode 100644 index 0000000..bb744db --- /dev/null +++ b/dags/business/business/community/total_price.py @@ -0,0 +1,97 @@ +""" +同步挂牌信息、户均总价至206 +""" +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor + +log = logging.getLogger(__name__) +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + + +def build_doc(item): + doc_dict = dict() + if 'three_months_sell_num' in item and pd.notna(item['three_months_sell_num']): + doc_dict['three_months_sell_num'] = int(item['three_months_sell_num']) + if 'total_price' in item and pd.notna(item['total_price']): + doc_dict['total_price'] = int(item['total_price']) + return doc_dict + + +def sync(city=None): + log.info(f"开始同步出售信息至206 {index_name}") + + seaweed_query = {'cat': 'district', 'status': 0} + if pd.notna(city): + seaweed_query['city'] = city + seaweed_df = pd.DataFrame( + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) + + # 加载库中均价数据 + community_summary_df = es206_client.scan_by_sql_to_data_frame("SELECT city, region, name, avg_price FROM community_summary") + + with ThreadPoolExecutor(50) as pool: + def deal(index, item): + if index % 1000 == 0: + log.info("已处理" + str(index)) + curr_city = item['city'] + curr_region = item['region'] + curr_name = item['name'] + names = [curr_name] + if type(item['alias']) == list: + for alias_item in item['alias']: + names.append(alias_item) + names_str = "','".join(names) + names_str = "'" + names_str + "'" + + # 近三个月挂牌量 + sample_num = commutils.get_sample(curr_city, curr_region, names_str, cat='sell', daysBefore=93) + seaweed_df.loc[index, 'three_months_sell_num'] = sample_num + + # 户均总价 + avg_price = community_summary_df.loc[(community_summary_df['city'] == curr_city) & (community_summary_df['region'] == curr_region) & (community_summary_df['name'] == curr_name)].reset_index().loc[0, 'avg_price'] + if pd.notna(avg_price): + min_avg_price = 0.8 * avg_price + max_avg_rice = 1.2 * avg_price + total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=93, min_avg_price=min_avg_price, max_avg_price=max_avg_rice) + seaweed_df.loc[index, 'total_price'] = total_price + + for index, item in seaweed_df.iterrows(): + pool.submit(deal, index, item) + + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + actions = list() + count = 0 + for index, item in seaweed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("已更新:" + str(count)) + except Exception as e: + print(e) + actions.clear() + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + print(e) + actions.clear() + log.info(f"完成同步出售信息至206 {index_name}")