优化相对价格排名

This commit is contained in:
shenjian@fangjia.com
2023-03-23 09:47:24 +08:00
parent 29d42f33d6
commit 0727376478
@@ -70,6 +70,7 @@ def build_doc(item):
find_count = 0
actions = list()
def find_around_price_item(seaweed):
if 'lng' not in seaweed:
return
@@ -78,26 +79,14 @@ def find_around_price_item(seaweed):
if pd.notna(lng):
city_now = seaweed['city']
around_seaweed_df = find_around_seaweed(city_now, lng, lat)
# 某些经纬度根本不在该城市下,数据问题
if len(around_seaweed_df) == 0:
return
mutex.acquire(timeout=300)
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
doc_info = build_doc(seaweed)
action = {
"_op_type": "update",
"_index": index_name,
"_id": seaweed['_id'],
"doc": doc_info,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
log.info("更新:" + str(find_count))
actions.clear()
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
@@ -117,6 +106,22 @@ def find_around_price_item(seaweed):
relative_price_ranking_score = 7
seaweed['credit_detail'] = {'around_community_num': around_community_num, 'relative_price_ranking_score': relative_price_ranking_score}
doc_info = build_doc(seaweed)
action = {
"_op_type": "update",
"_index": index_name,
"_id": seaweed['_id'],
"doc": doc_info,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
log.info("更新:" + str(find_count))
actions.clear()
mutex.release()
@@ -131,15 +136,16 @@ def sync(city=None):
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
with ThreadPoolExecutor(max_workers=50) as pool:
for seaweed in seaweed_list:
pool.submit(find_around_price_item, seaweed)
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
log.info("完成同步相对价格排名数据...")