From ee8d87b95785cc87f77b1fc5fa1298b344190070 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Mon, 24 Oct 2022 17:24:52 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/community/community_summary.py | 14 +- .../community/relative_price_ranking.py | 129 ++++++++++++++++++ 2 files changed, 136 insertions(+), 7 deletions(-) create mode 100644 dags/business/business/community/relative_price_ranking.py diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index 9d4e2ad..b96c34b 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -17,12 +17,12 @@ log.info("同步类型:" + type) if type == 'all': - comm.sync('上海') - #price.sync() - #sell.sync() - #avg_cost.sync() - #trade.sync() + comm.sync() + price.sync() + sell.sync() + avg_cost.sync() + trade.sync() total_price.sync() - #traffic.sync() - #medical.sync() + traffic.sync() + medical.sync() life.sync() diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py new file mode 100644 index 0000000..09f2cab --- /dev/null +++ b/dags/business/business/community/relative_price_ranking.py @@ -0,0 +1,129 @@ +""" +相对价格排名计算同步至206 +""" +import logging +import pandas as pd +from goojia_utils import ElasticSearchClient, commutils, dateutils +from elasticsearch import helpers +import numpy as np +from concurrent.futures import ThreadPoolExecutor + +log = logging.getLogger(__name__) +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + + +def find_around_seaweed(lng, lat): + """ + 查找周边均价 + """ + if pd.isna(lng): + return None + ret_es = es206_client.elastic_client.search({ + "query": { + "bool": { + "filter": [ + { + "geo_distance": { + "distance": 1000, + "location": [lng, lat] + } + } + ] + } + } + }, index='community_summary') + return ret_es['aggregations']['price']['value'] + + +def build_doc(item): + doc_dict = dict() + if 'avg_cost' in item and pd.notna(item['avg_cost']): + doc_dict['avg_cost'] = item['avg_cost'] + return doc_dict + + +def update_status(new_df): + actions = list() + + def deal(item): + action_doc = { + "_op_type": "update", + "_index": "bd_poi", + "_id": item['_id'], + "doc": {'status': 0} + } + actions.append(action_doc) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + return item + + new_df.apply(lambda x: deal(x), axis=1) + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + +def sync(): + log.info("开始同步消费数据...") + df = es206_client.scan_by_sql_to_data_frame( + "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')") + + # 按百分位5-95筛选数据 + cities = df['city'].unique() + for city in cities: + df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0) + df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0) + + new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])] + new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean) + + # 按最新的百分位更新库中价格状态 + update_status(new_df) + + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + + def find_around_price_item(index, item): + if 'lng' not in item: + return + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + price = find_around_price(lng, lat) + seaweed_df.loc[index, 'avg_cost'] = price + + with ThreadPoolExecutor(max_workers=50) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(find_around_price_item, index, item) + + actions = list() + count = 0 + for index, item in seaweed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + except Exception as e: + log.error(e) + actions.clear() + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + log.error(e) + actions.clear() + + log.info("完成同步消费数据...") +