diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index b6f72f1..045e064 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -9,28 +9,57 @@ from concurrent.futures import ThreadPoolExecutor import sys from threading import Lock -# 创建线程锁 mutex = Lock() - log = logging.getLogger(__name__) es206_client = ElasticSearchClient(host='192.168.0.206') index_name = "community_summary" m_date = dateutils.iso_format() path = "/home/airflow/data/community" +file_path = "/opt/airflow/dags/repo/dags/business/business/community/data" + +avg_cost_min_max_df = pd.read_excel(file_path + '/全国各区域人均消费中位值.xlsx') +avg_cost_city_region_dict = dict() -def find_around_price(city_now, lng, lat): +# 加载各区域人均消费最大最小值 +def deal_avg_cost(x): + city_now = x['city'] + region_now = x['region'] + if pd.notna(city_now) and pd.notna(region_now): + avg_cost_city_region_dict[city_now + region_now] = {'min': x['3'], 'max': x['97']} + elif pd.notna(city_now): + avg_cost_city_region_dict[city_now] = {'min': x['3'], 'max': x['97']} + + +avg_cost_min_max_df.apply(lambda x: deal_avg_cost(x), axis=1) + + +def find_around_price(seaweed): """ 查找周边均价 返回取值为10~300,为所有有价格的1-99百分位 """ - if pd.isna(lng): - return None + lng = commutils.format_lat_lng(seaweed['lng']) + lat = commutils.format_lat_lng(seaweed['lat']) + city_short = seaweed['city'] + region_short = seaweed['region'] + city_display = areautils.get_city_display(city_short) + filter_query = [{"terms": {"tag": ["美食;中餐厅", "美食;外国餐厅"]}}] - if pd.notna(city_now): - filter_query.append({"term": {"city": {"value": city_now}}}) - filter_query.append({"range": {"price": {"gte": 10, "lte": 300}}}) + if pd.notna(city_display): + filter_query.append({"term": {"city": {"value": city_display}}}) + + min_now = 10 + max_now = 300 + if (city_short+region_short) in avg_cost_city_region_dict: + min_now = avg_cost_city_region_dict[city_short + region_short]['min'] + max_now = avg_cost_city_region_dict[city_short + region_short]['max'] + elif city_short in avg_cost_city_region_dict: + min_now = avg_cost_city_region_dict[city_short]['min'] + max_now = avg_cost_city_region_dict[city_short]['max'] + + filter_query.append({"range": {"price": {"gte": min_now, "lte": max_now}}}) filter_query.append({"geo_distance": {"distance": 1000, "location": [lng, lat]}}) ret_es = es206_client.elastic_client.search({ @@ -61,20 +90,22 @@ def build_doc(item): return doc_dict -def find_around_price_item(seaweed_df, index, item): - if index % 1000 == 0: - log.info("已查找" + str(index)) - if 'lng' not in item: +find_count = 0 + + +def find_around_price_item(seaweed): + global find_count + find_count += 1 + if find_count % 1000 == 0: + log.info("已查找" + str(find_count)) + if 'lng' not in seaweed: return - lng = commutils.format_lat_lng(item['lng']) - lat = commutils.format_lat_lng(item['lat']) - city_now = areautils.get_city_display(item['city']) - price = find_around_price(city_now, lng, lat) + price = find_around_price(seaweed) mutex.acquire(timeout=60) if pd.notna(price): - seaweed_df.loc[index, 'avg_cost'] = int(price) + seaweed['avg_cost'] = int(price) else: - seaweed_df.loc[index, 'avg_cost'] = None + seaweed['avg_cost'] = None mutex.release() @@ -85,11 +116,17 @@ def sync(city=None): seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() seaweed_df = seaweed_df.reset_index(drop=True) + seaweed_list = list() + seaweed_df.apply(lambda x: seaweed_list.append(dict(x)), axis=1) + with ThreadPoolExecutor(max_workers=30) as pool: - for index, item in seaweed_df.iterrows(): - pool.submit(find_around_price_item, seaweed_df, index, item) - # for index, item in seaweed_df[seaweed_df['_id'] == '50bd172ae4b0fbe60a6fe628'].iterrows(): - # find_around_price_item(index, item) + for seaweed in seaweed_list: + pool.submit(find_around_price_item, seaweed) + + # for seaweed in seaweed_list: + # find_around_price_item(seaweed) + + seaweed_df = pd.DataFrame(seaweed_list) actions = list() count = 0 for index, item in seaweed_df.iterrows(): diff --git a/dags/business/business/community/data/全国各区域人均消费中位值.xlsx b/dags/business/business/community/data/全国各区域人均消费中位值.xlsx new file mode 100644 index 0000000..3935a59 Binary files /dev/null and b/dags/business/business/community/data/全国各区域人均消费中位值.xlsx differ