优化价格同步
This commit is contained in:
@@ -0,0 +1,150 @@
|
||||
"""
|
||||
小区周边消费同步至206
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
seaweed_conn = MAIN_MONGO_CLIENT.get_collection('fangjia', 'seaweed')
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
|
||||
def find_around_price(lng, lat):
|
||||
"""
|
||||
查找周边均价
|
||||
"""
|
||||
if pd.isna(lng):
|
||||
return None
|
||||
ret_es = es206_client.elastic_client.search({
|
||||
"query": {
|
||||
"bool": {
|
||||
"must": [
|
||||
{
|
||||
"term": {
|
||||
"status": {
|
||||
"value": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"filter": [
|
||||
{
|
||||
"geo_distance": {
|
||||
"distance": 1000,
|
||||
"location": [lng, lat]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
"size": 0,
|
||||
"aggs": {
|
||||
"price": {
|
||||
"avg": {
|
||||
"field": "price"
|
||||
}
|
||||
}
|
||||
}
|
||||
}, index='bd_poi')
|
||||
return ret_es['aggregations']['price']['value']
|
||||
|
||||
|
||||
def build_doc(item):
|
||||
doc_dict = dict()
|
||||
if 'avg_cost' in item and pd.notna(item['avg_cost']):
|
||||
doc_dict['avg_cost'] = item['avg_cost']
|
||||
return doc_dict
|
||||
|
||||
|
||||
def update_status(new_df):
|
||||
actions = list()
|
||||
|
||||
def deal(item):
|
||||
action_doc = {
|
||||
"_op_type": "update",
|
||||
"_index": "bd_poi",
|
||||
"_id": item['_id'],
|
||||
"doc": {'status': 0}
|
||||
}
|
||||
actions.append(action_doc)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
return item
|
||||
|
||||
new_df.apply(lambda x: deal(x), axis=1)
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
def sync():
|
||||
log.info("开始同步消费数据...")
|
||||
df = es206_client.scan_by_sql_to_data_frame(
|
||||
"select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')")
|
||||
|
||||
# 按百分位5-95筛选数据
|
||||
cities = df['city'].unique()
|
||||
for city in cities:
|
||||
df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0)
|
||||
df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0)
|
||||
|
||||
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
|
||||
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
|
||||
|
||||
# 按最新的百分位更新库中价格状态
|
||||
update_status(new_df)
|
||||
|
||||
seaweed_df = pd.DataFrame(
|
||||
list(seaweed_conn.find({'cat': 'district', 'status': 0}, {'city': 1, 'lng': 1, 'lat': 1}).batch_size(100000)))
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
return
|
||||
lng = commutils.format_lat_lng(item['lng'])
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
price = find_around_price(lng, lat)
|
||||
seaweed_df.loc[index, 'avg_cost'] = price
|
||||
|
||||
with ThreadPoolExecutor(max_workers=50) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(find_around_price_item, index, item)
|
||||
|
||||
seaweed_df['_id'] = seaweed_df['_id'].astype(str)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
doc_info = build_doc(item)
|
||||
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info
|
||||
}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
actions.clear()
|
||||
|
||||
log.info("完成同步消费数据...")
|
||||
|
||||
@@ -4,6 +4,7 @@ import comm
|
||||
import price
|
||||
import sell
|
||||
import trade
|
||||
import avg_cost
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -12,7 +13,8 @@ log.info("同步类型:" + type)
|
||||
|
||||
|
||||
if type == 'all':
|
||||
comm.sync()
|
||||
price.sync()
|
||||
#comm.sync()
|
||||
#price.sync()
|
||||
sell.sync()
|
||||
trade.sync()
|
||||
#avg_cost.sync()
|
||||
#trade.sync()
|
||||
@@ -36,8 +36,10 @@ def sync(city=None):
|
||||
# 加载库中均价数据
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame("SELECT city, region, name, avg_price FROM community_summary")
|
||||
|
||||
with ThreadPoolExecutor(30) as pool:
|
||||
with ThreadPoolExecutor(50) as pool:
|
||||
def deal(index, item):
|
||||
if index % 1000 == 0:
|
||||
log.info("已处理" + str(index))
|
||||
curr_city = item['city']
|
||||
curr_region = item['region']
|
||||
curr_name = item['name']
|
||||
@@ -82,7 +84,7 @@ def sync(city=None):
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
print("已更新:" + str(count))
|
||||
except Exception as e:
|
||||
print(e)
|
||||
actions.clear()
|
||||
|
||||
Reference in New Issue
Block a user