优化
This commit is contained in:
@@ -17,12 +17,12 @@ log.info("同步类型:" + type)
|
||||
|
||||
|
||||
if type == 'all':
|
||||
comm.sync('上海')
|
||||
#price.sync()
|
||||
#sell.sync()
|
||||
#avg_cost.sync()
|
||||
#trade.sync()
|
||||
comm.sync()
|
||||
price.sync()
|
||||
sell.sync()
|
||||
avg_cost.sync()
|
||||
trade.sync()
|
||||
total_price.sync()
|
||||
#traffic.sync()
|
||||
#medical.sync()
|
||||
traffic.sync()
|
||||
medical.sync()
|
||||
life.sync()
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""
|
||||
相对价格排名计算同步至206
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import ElasticSearchClient, commutils, dateutils
|
||||
from elasticsearch import helpers
|
||||
import numpy as np
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
es206_client = ElasticSearchClient(host='192.168.0.206')
|
||||
|
||||
index_name = "community_summary"
|
||||
m_date = dateutils.iso_format()
|
||||
|
||||
|
||||
def find_around_seaweed(lng, lat):
|
||||
"""
|
||||
查找周边均价
|
||||
"""
|
||||
if pd.isna(lng):
|
||||
return None
|
||||
ret_es = es206_client.elastic_client.search({
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"geo_distance": {
|
||||
"distance": 1000,
|
||||
"location": [lng, lat]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}, index='community_summary')
|
||||
return ret_es['aggregations']['price']['value']
|
||||
|
||||
|
||||
def build_doc(item):
|
||||
doc_dict = dict()
|
||||
if 'avg_cost' in item and pd.notna(item['avg_cost']):
|
||||
doc_dict['avg_cost'] = item['avg_cost']
|
||||
return doc_dict
|
||||
|
||||
|
||||
def update_status(new_df):
|
||||
actions = list()
|
||||
|
||||
def deal(item):
|
||||
action_doc = {
|
||||
"_op_type": "update",
|
||||
"_index": "bd_poi",
|
||||
"_id": item['_id'],
|
||||
"doc": {'status': 0}
|
||||
}
|
||||
actions.append(action_doc)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
return item
|
||||
|
||||
new_df.apply(lambda x: deal(x), axis=1)
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
def sync():
|
||||
log.info("开始同步消费数据...")
|
||||
df = es206_client.scan_by_sql_to_data_frame(
|
||||
"select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')")
|
||||
|
||||
# 按百分位5-95筛选数据
|
||||
cities = df['city'].unique()
|
||||
for city in cities:
|
||||
df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0)
|
||||
df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0)
|
||||
|
||||
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
|
||||
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
|
||||
|
||||
# 按最新的百分位更新库中价格状态
|
||||
update_status(new_df)
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
return
|
||||
lng = commutils.format_lat_lng(item['lng'])
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
price = find_around_price(lng, lat)
|
||||
seaweed_df.loc[index, 'avg_cost'] = price
|
||||
|
||||
with ThreadPoolExecutor(max_workers=50) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(find_around_price_item, index, item)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
doc_info = build_doc(item)
|
||||
|
||||
action = {
|
||||
"_op_type": "update",
|
||||
"_index": index_name,
|
||||
"_id": item['_id'],
|
||||
"doc": doc_info
|
||||
}
|
||||
|
||||
actions.append(action)
|
||||
if len(actions) == 5000:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
log.info("更新:" + str(count))
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
try:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
except Exception as e:
|
||||
log.error(e)
|
||||
actions.clear()
|
||||
|
||||
log.info("完成同步消费数据...")
|
||||
|
||||
Reference in New Issue
Block a user