From 0d8bb6acbeccdde826a57f8491dee4bde74ab9c7 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 18 Oct 2022 10:50:11 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=88=90=E4=BA=A4=E4=BF=A1?= =?UTF-8?q?=E6=81=AF=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/community_summary.py | 4 +- dags/business/business/community/sell.py | 3 +- dags/business/business/community/trade.py | 71 +++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) create mode 100644 dags/business/business/community/trade.py diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index e3e63b8..d5cda3b 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -3,6 +3,7 @@ import sys import comm import price import sell +import trade log = logging.getLogger(__name__) @@ -13,4 +14,5 @@ log.info("同步类型:" + type) if type == 'all': comm.sync() price.sync() - sell.sync() \ No newline at end of file + sell.sync() + trade.sync() \ No newline at end of file diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index b7dc941..123c7f1 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -1,5 +1,5 @@ """ -同步挂牌、成交信息至206 +同步挂牌信息至206 """ import logging import pandas as pd @@ -21,6 +21,7 @@ def build_doc(item): doc_dict['three_months_sell_num'] = int(item['three_months_sell_num']) return doc_dict + def sync(city=None): log.info(f"开始同步出售信息至206 {index_name}") diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py new file mode 100644 index 0000000..282f2d7 --- /dev/null +++ b/dags/business/business/community/trade.py @@ -0,0 +1,71 @@ +""" +同步成交信息至206 +""" +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils, areautils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor + +log = logging.getLogger(__name__) +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + + +def build_doc(item): + doc_dict = dict() + if pd.notna(item['three_month_trade_avg_price']): + doc_dict['three_month_trade_avg_price'] = int(item['three_month_trade_avg_price']) + return doc_dict + + +def sync(city=None): + log.info(f"开始同步成交信息至206 {index_name}") + + seaweed_query = {'cat': 'district', 'status': 0} + cities = None + if pd.notna(city): + seaweed_query['city'] = city + cities = [city] + seaweed_df = pd.DataFrame( + list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'alias': 1}).batch_size(100000))) + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + deal_price_df = commutils.get_deal_avg_price(cities, 3) + deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] + deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name', 'avg_price': 'three_month_trade_avg_price'}) + + seaweed_df = pd.merge(seaweed_df, deal_price_df, how='left', on=['city', 'region', 'name']) + + + actions = list() + count = 0 + for index, item in seaweed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + print("更新:" + str(count)) + except Exception as e: + print(e) + actions.clear() + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + print(e) + actions.clear() + log.info(f"完成同步成交信息至206 {index_name}") \ No newline at end of file