优化价格同步

This commit is contained in:
shenjian@fangjia.com
2022-10-18 15:15:38 +08:00
parent ff375e1012
commit 959d320baa
3 changed files with 159 additions and 5 deletions
@@ -0,0 +1,150 @@
"""
小区周边消费同步至206
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from elasticsearch import helpers
import numpy as np
from concurrent.futures import ThreadPoolExecutor
log = logging.getLogger(__name__)
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
m_date = dateutils.iso_format()
def find_around_price(lng, lat):
"""
查找周边均价
"""
if pd.isna(lng):
return None
ret_es = es206_client.elastic_client.search({
"query": {
"bool": {
"must": [
{
"term": {
"status": {
"value": 0
}
}
}
],
"filter": [
{
"geo_distance": {
"distance": 1000,
"location": [lng, lat]
}
}
]
}
},
"size": 0,
"aggs": {
"price": {
"avg": {
"field": "price"
}
}
}
}, index='bd_poi')
return ret_es['aggregations']['price']['value']
def build_doc(item):
doc_dict = dict()
if 'avg_cost' in item and pd.notna(item['avg_cost']):
doc_dict['avg_cost'] = item['avg_cost']
return doc_dict
def update_status(new_df):
actions = list()
def deal(item):
action_doc = {
"_op_type": "update",
"_index": "bd_poi",
"_id": item['_id'],
"doc": {'status': 0}
}
actions.append(action_doc)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
return item
new_df.apply(lambda x: deal(x), axis=1)
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def sync():
log.info("开始同步消费数据...")
df = es206_client.scan_by_sql_to_data_frame(
"select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')")
# 按百分位5-95筛选数据
cities = df['city'].unique()
for city in cities:
df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0)
df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0)
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.DataFrame(
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'lng': 1, 'lat': 1}).batch_size(100000)))
def find_around_price_item(index, item):
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
price = find_around_price(lng, lat)
seaweed_df.loc[index, 'avg_cost'] = price
with ThreadPoolExecutor(max_workers=50) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
doc_info = build_doc(item)
action = {
"_op_type": "update",
"_index": index_name,
"_id": item['_id'],
"doc": doc_info
}
actions.append(action)
if len(actions) == 5000:
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
except Exception as e:
log.error(e)
actions.clear()
if len(actions) > 0:
try:
helpers.bulk(es206_client.elastic_client, actions)
except Exception as e:
log.error(e)
actions.clear()
log.info("完成同步消费数据...")
@@ -4,6 +4,7 @@ import comm
import price
import sell
import trade
import avg_cost
log = logging.getLogger(__name__)
@@ -12,7 +13,8 @@ log.info("同步类型:" + type)
if type == 'all':
comm.sync()
price.sync()
#comm.sync()
#price.sync()
sell.sync()
trade.sync()
#avg_cost.sync()
#trade.sync()
+4 -2
View File
@@ -36,8 +36,10 @@ def sync(city=None):
# 加载库中均价数据
community_summary_df = es206_client.scan_by_sql_to_data_frame("SELECT city, region, name, avg_price FROM community_summary")
with ThreadPoolExecutor(30) as pool:
with ThreadPoolExecutor(50) as pool:
def deal(index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
curr_city = item['city']
curr_region = item['region']
curr_name = item['name']
@@ -82,7 +84,7 @@ def sync(city=None):
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
print("更新:" + str(count))
print("已更新:" + str(count))
except Exception as e:
print(e)
actions.clear()