优化相对价格排名性能

This commit is contained in:
shenjian@fangjia.com
2023-03-10 17:37:21 +08:00
parent c9e51dd426
commit 71d00d28dd
@@ -18,6 +18,7 @@ path = "/home/airflow/data/community"
m_date = dateutils.iso_format()
mutex = Lock()
def find_around_seaweed(city_now, lng, lat):
"""
查找周边均价
@@ -30,11 +31,11 @@ def find_around_seaweed(city_now, lng, lat):
"bool": {
"filter": [
{
"term": {
"city": {
"value": city_now
}
}
"term": {
"city": {
"value": city_now
}
}
},
{
"range": {
@@ -63,6 +64,31 @@ def build_doc(item):
return doc_dict
find_count = 0
def find_around_price_item(seaweed):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
if 'lng' not in seaweed:
return
lng = commutils.format_lat_lng(seaweed['lng'])
lat = commutils.format_lat_lng(seaweed['lat'])
if pd.notna(lng):
city_now = seaweed['city']
around_seaweed_df = find_around_seaweed(city_now, lng, lat)
mutex.acquire(timeout=60)
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed['relative_price_ranking'] = around_seaweed_df[around_seaweed_df['id'] == seaweed['_id']].iloc[0][
'relative_price_ranking']
elif len(around_seaweed_df) == 1:
seaweed['relative_price_ranking'] = 0.5
mutex.release()
def sync(city=None):
log.info("开始同步相对价格排名数据...")
@@ -71,30 +97,17 @@ def sync(city=None):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
def find_around_price_item(index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
if pd.notna(lng):
city_now = item['city']
around_seaweed_df = find_around_seaweed(city_now, lng, lat)
mutex.acquire(timeout=60)
if len(around_seaweed_df) > 1:
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
elif len(around_seaweed_df) == 1:
seaweed_df.loc[index, 'relative_price_ranking'] = 0.5
mutex.release()
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, index, item)
for seaweed in seaweed_list:
pool.submit(find_around_price_item, seaweed)
# for index, item in seaweed_df.iterrows():
# find_around_price_item(index, item)
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
seaweed_df = pd.DataFrame(seaweed_list)
actions = list()
count = 0
@@ -105,17 +118,15 @@ def sync(city=None):
"_op_type": "update",
"_index": index_name,
"_id": item['_id'],
"doc": doc_info
"doc": doc_info,
"_retry_on_conflict": 5
}
actions.append(action)
if len(actions) == 5000:
try:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
except Exception as e:
log.error(e)
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
log.info("更新:" + str(count))
actions.clear()
if len(actions) > 0:
try:
@@ -128,4 +139,4 @@ def sync(city=None):
city = sys.argv[1] if len(sys.argv) > 1 else None
sync(city)
sync(city)