人均消费区域级别极值过滤

This commit is contained in:
shenjian@fangjia.com
2023-03-10 16:56:19 +08:00
parent 98466f4b22
commit c9e51dd426
2 changed files with 59 additions and 22 deletions
+59 -22
View File
@@ -9,28 +9,57 @@ from concurrent.futures import ThreadPoolExecutor
import sys
from threading import Lock
# 创建线程锁
mutex = Lock()
log = logging.getLogger(__name__)
es206_client = ElasticSearchClient(host='192.168.0.206')
index_name = "community_summary"
m_date = dateutils.iso_format()
path = "/home/airflow/data/community"
file_path = "/opt/airflow/dags/repo/dags/business/business/community/data"
avg_cost_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx')
avg_cost_city_region_dict = dict()
def find_around_price(city_now, lng, lat):
# 加载各区域人均消费最大最小值
def deal_avg_cost(x):
city_now = x['city']
region_now = x['region']
if pd.notna(city_now) and pd.notna(region_now):
avg_cost_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']}
elif pd.notna(city_now):
avg_cost_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']}
avg_cost_min_max_df.apply(lambda x: deal_avg_cost(x), axis=1)
def find_around_price(seaweed):
"""
查找周边均价
返回取值为10~300,为所有有价格的1-99百分位
"""
if pd.isna(lng):
return None
lng = commutils.format_lat_lng(seaweed['lng'])
lat = commutils.format_lat_lng(seaweed['lat'])
city_short = seaweed['city']
region_short = seaweed['region']
city_display = areautils.get_city_display(city_short)
filter_query = [{"terms": {"tag": ["美食;中餐厅", "美食;外国餐厅"]}}]
if pd.notna(city_now):
filter_query.append({"term": {"city": {"value": city_now}}})
filter_query.append({"range": {"price": {"gte": 10, "lte": 300}}})
if pd.notna(city_display):
filter_query.append({"term": {"city": {"value": city_display}}})
min_now = 10
max_now = 300
if (city_short+region_short) in avg_cost_city_region_dict:
min_now = avg_cost_city_region_dict[city_short + region_short]['min']
max_now = avg_cost_city_region_dict[city_short + region_short]['max']
elif city_short in avg_cost_city_region_dict:
min_now = avg_cost_city_region_dict[city_short]['min']
max_now = avg_cost_city_region_dict[city_short]['max']
filter_query.append({"range": {"price": {"gte": min_now, "lte": max_now}}})
filter_query.append({"geo_distance": {"distance": 1000, "location": [lng, lat]}})
ret_es = es206_client.elastic_client.search({
@@ -61,20 +90,22 @@ def build_doc(item):
return doc_dict
def find_around_price_item(seaweed_df, index, item):
if index % 1000 == 0:
log.info("已查找" + str(index))
if 'lng' not in item:
find_count = 0
def find_around_price_item(seaweed):
global find_count
find_count += 1
if find_count % 1000 == 0:
log.info("已查找" + str(find_count))
if 'lng' not in seaweed:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
city_now = areautils.get_city_display(item['city'])
price = find_around_price(city_now, lng, lat)
price = find_around_price(seaweed)
mutex.acquire(timeout=60)
if pd.notna(price):
seaweed_df.loc[index, 'avg_cost'] = int(price)
seaweed['avg_cost'] = int(price)
else:
seaweed_df.loc[index, 'avg_cost'] = None
seaweed['avg_cost'] = None
mutex.release()
@@ -85,11 +116,17 @@ def sync(city=None):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
seaweed_list = list()
seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1)
with ThreadPoolExecutor(max_workers=30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(find_around_price_item, seaweed_df, index, item)
# for index, item in seaweed_df[seaweed_df['_id'] == '50bd172ae4b0fbe60a6fe628'].iterrows():
# find_around_price_item(index, item)
for seaweed in seaweed_list:
pool.submit(find_around_price_item, seaweed)
# for seaweed in seaweed_list:
# find_around_price_item(seaweed)
seaweed_df = pd.DataFrame(seaweed_list)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():