From 92b9a3d2792066c9263b7d545153f4198713ac99 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 18 Oct 2022 15:15:38 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9=E6=B6=88?= =?UTF-8?q?=E8=B4=B9=E6=95=B0=E6=8D=AE=E5=90=8C=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/business/business/community/avg_cost.py | 150 ++++++++++++++++++ .../business/community/community_summary.py | 8 +- 2 files changed, 155 insertions(+), 3 deletions(-) create mode 100644 dags/business/business/community/avg_cost.py diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py new file mode 100644 index 0000000..72561c8 --- /dev/null +++ b/dags/business/business/community/avg_cost.py @@ -0,0 +1,150 @@ +""" +小区周边消费同步至206 +""" +import logging +import pandas as pd +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from elasticsearch import helpers +import numpy as np +from concurrent.futures import ThreadPoolExecutor + +log = logging.getLogger(__name__) +seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed') +es206_client = ElasticSearchClient(host='192.168.0.206') + +index_name = "community_summary" +m_date = dateutils.iso_format() + + +def find_around_price(lng, lat): + """ + 查找周边均价 + """ + if pd.isna(lng): + return None + ret_es = es206_client.elastic_client.search({ + "query": { + "bool": { + "must": [ + { + "term": { + "status": { + "value": 0 + } + } + } + ], + "filter": [ + { + "geo_distance": { + "distance": 1000, + "location": [lng, lat] + } + } + ] + } + }, + "size": 0, + "aggs": { + "price": { + "avg": { + "field": "price" + } + } + } + }, index='bd_poi') + return ret_es['aggregations']['price']['value'] + + +def build_doc(item): + doc_dict = dict() + if 'avg_cost' in item and pd.notna(item['avg_cost']): + doc_dict['avg_cost'] = item['avg_cost'] + return doc_dict + + +def update_status(new_df): + actions = list() + + def deal(item): + action_doc = { + "_op_type": "update", + "_index": "bd_poi", + "_id": item['_id'], + "doc": {'status': 0} + } + actions.append(action_doc) + if len(actions) == 5000: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + return item + + new_df.apply(lambda x: deal(x), axis=1) + if len(actions) > 0: + helpers.bulk(es206_client.elastic_client, actions) + actions.clear() + +def sync(): + log.info("开始同步消费数据...") + df = es206_client.scan_by_sql_to_data_frame( + "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')") + + # 按百分位5-95筛选数据 + cities = df['city'].unique() + for city in cities: + df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0) + df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0) + + new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])] + new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean) + + # 按最新的百分位更新库中价格状态 + update_status(new_df) + + seaweed_df = pd.DataFrame( + list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'lng': 1, 'lat': 1}).batch_size(100000))) + + def find_around_price_item(index, item): + if 'lng' not in item: + return + lng = commutils.format_lat_lng(item['lng']) + lat = commutils.format_lat_lng(item['lat']) + price = find_around_price(lng, lat) + seaweed_df.loc[index, 'avg_cost'] = price + + with ThreadPoolExecutor(max_workers=50) as pool: + for index, item in seaweed_df.iterrows(): + pool.submit(find_around_price_item, index, item) + + seaweed_df['_id'] = seaweed_df['_id'].astype(str) + + actions = list() + count = 0 + for index, item in seaweed_df.iterrows(): + doc_info = build_doc(item) + + action = { + "_op_type": "update", + "_index": index_name, + "_id": item['_id'], + "doc": doc_info + } + + actions.append(action) + if len(actions) == 5000: + try: + helpers.bulk(es206_client.elastic_client, actions) + count += len(actions) + log.info("更新:" + str(count)) + except Exception as e: + log.error(e) + actions.clear() + if len(actions) > 0: + try: + helpers.bulk(es206_client.elastic_client, actions) + except Exception as e: + log.error(e) + actions.clear() + + log.info("完成同步消费数据...") + diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index d5cda3b..de7e283 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -4,6 +4,7 @@ import comm import price import sell import trade +import avg_cost log = logging.getLogger(__name__) @@ -12,7 +13,8 @@ log.info("同步类型:" + type) if type == 'all': - comm.sync() - price.sync() + #comm.sync() + #price.sync() sell.sync() - trade.sync() \ No newline at end of file + #avg_cost.sync() + #trade.sync() \ No newline at end of file