From 0fa6b8821f1bf7b5dafc63346ad0a2041c9179f2 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Wed, 10 May 2023 15:10:24 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/relative_price_fix.py | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 dags/business/business/community/relative_price_fix.py diff --git a/dags/business/business/community/relative_price_fix.py b/dags/business/business/community/relative_price_fix.py new file mode 100644 index 0000000..712441b --- /dev/null +++ b/dags/business/business/community/relative_price_fix.py @@ -0,0 +1,69 @@ +""" +修正相对价格排名 +""" +import logging +import pandas as pd +from goojia_utils import ElasticSearchClient, commutils, dateutils, data_utils +from elasticsearch import helpers +from concurrent.futures import ThreadPoolExecutor +import sys +from threading import Lock + +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +path = "/home/airflow/data/community" +m_date = dateutils.iso_format() +mutex = Lock() + + +def sync(city=None): + log.info("开始调整相对价格排名数据...") + + query_filter = [] + if pd.notna(city): + query_filter.append({"term": {"city": city}}) + + query = { + "_source": ["city", "region", "relative_price_ranking", "credit_detail"], + "query": { + "bool": { + "filter": query_filter + } + } + } + community_summary_origin_df = es206_client.scan_to_data_frame("community_summary", query) + community_summary_df = community_summary_origin_df[community_summary_origin_df['credit_detail'].notna()].reset_index(drop=True) + community_summary_df = data_utils.flatten_json(community_summary_df, 'credit_detail') + community_summary_df['around_community_max_num'] = community_summary_df.groupby(by=['city'])['around_community_num'].transform('max') + community_summary_df['relative_price_ranking_fix'] = community_summary_df['around_community_num'] / community_summary_df['around_community_max_num'] * community_summary_df['relative_price_ranking'] + + actions = list() + count = 0 + for index, item in community_summary_df[community_summary_df['relative_price_ranking_fix'].notna()].iterrows(): + doc_info = {'relative_price_ranking_fix': item['relative_price_ranking_fix']} + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info, + "_retry_on_conflict": 5 + } + + actions.append(action) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + actions.clear() + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + + log.info("完成调整相对价格排名数据...") + + +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +sync(sys_city)