diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index b78c8ed..153481d 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -100,7 +100,7 @@ def sync(): # 按最新的百分位更新库中价格状态 update_status(new_df) - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') def find_around_price_item(index, item): if 'lng' not in item: diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index f426694..51cf489 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -94,7 +94,7 @@ def sync(city=None): if len(actions) == 5000: helpers.bulk(es206_client.elastic_client, actions) count += len(actions) - print("更新:" + str(count)) + log.info("更新:" + str(count)) actions.clear() # 对于不存在的插入 @@ -113,12 +113,12 @@ def sync(city=None): if len(actions) == 5000: helpers.bulk(es206_client.elastic_client, actions) count += len(actions) - print("插入:" + str(count)) + log.info("插入:" + str(count)) actions.clear() if len(actions) > 0: helpers.bulk(es206_client.elastic_client, actions) actions.clear() # 写入csv文件,防止小区变动导致的后续过程更新报错 - seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False) + seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False) log.info(f"完成同步小区基本属性至206 {index_name}") diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py index b96c34b..8d4c643 100644 --- a/dags/business/business/community/community_summary.py +++ b/dags/business/business/community/community_summary.py @@ -9,6 +9,7 @@ import total_price import traffic import medical import life +import relative_price_ranking log = logging.getLogger(__name__) @@ -26,3 +27,4 @@ if type == 'all': traffic.sync() medical.sync() life.sync() + relative_price_ranking.sync() diff --git a/dags/business/business/community/data/seaweed.csv b/dags/business/business/community/data/seaweed.csv new file mode 100644 index 0000000..e69de29 diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 7a82f88..b09f6aa 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -79,7 +79,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index 2271174..cd07ddb 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -67,7 +67,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') actions = list() with ThreadPoolExecutor(max_workers=30) as pool: diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index a4befa1..170dae9 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -42,7 +42,7 @@ def sync(city=None): cities = [city] else: cities = None - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') # 小区所在区域均价 region_price_df = commutils.get_region_price(city=city) diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index 09f2cab..1e70d61 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -1,5 +1,6 @@ """ 相对价格排名计算同步至206 +即周边一公里内小区价格排名0-1之间 """ import logging import pandas as pd @@ -21,10 +22,18 @@ def find_around_seaweed(lng, lat): """ if pd.isna(lng): return None - ret_es = es206_client.elastic_client.search({ + ret_es = es206_client.scan_to_data_frame('community_summary', { + "_source": ["id", "avg_price"], "query": { "bool": { "filter": [ + { + "range": { + "avg_price": { + "gt": 0 + } + } + }, { "geo_distance": { "distance": 1000, @@ -34,64 +43,31 @@ def find_around_seaweed(lng, lat): ] } } - }, index='community_summary') - return ret_es['aggregations']['price']['value'] + }) + return ret_es def build_doc(item): doc_dict = dict() - if 'avg_cost' in item and pd.notna(item['avg_cost']): - doc_dict['avg_cost'] = item['avg_cost'] + if 'relative_price_ranking' in item and pd.notna(item['relative_price_ranking']): + doc_dict['relative_price_ranking'] = item['relative_price_ranking'] return doc_dict -def update_status(new_df): - actions = list() - - def deal(item): - action_doc = { - "_op_type": "update", - "_index": "bd_poi", - "_id": item['_id'], - "doc": {'status': 0} - } - actions.append(action_doc) - if len(actions) == 5000: - helpers.bulk(es206_client.elastic_client, actions) - actions.clear() - return item - - new_df.apply(lambda x: deal(x), axis=1) - if len(actions) > 0: - helpers.bulk(es206_client.elastic_client, actions) - actions.clear() - def sync(): - log.info("开始同步消费数据...") - df = es206_client.scan_by_sql_to_data_frame( - "select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')") + log.info("开始同步相对价格排名数据...") - # 按百分位5-95筛选数据 - cities = df['city'].unique() - for city in cities: - df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0) - df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0) - - new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])] - new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean) - - # 按最新的百分位更新库中价格状态 - update_status(new_df) - - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') def find_around_price_item(index, item): if 'lng' not in item: return lng = commutils.format_lat_lng(item['lng']) lat = commutils.format_lat_lng(item['lat']) - price = find_around_price(lng, lat) - seaweed_df.loc[index, 'avg_cost'] = price + if pd.notna(lng): + around_seaweed_df = find_around_seaweed(lng, lat) + around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True) + seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0] with ThreadPoolExecutor(max_workers=50) as pool: for index, item in seaweed_df.iterrows(): @@ -125,5 +101,5 @@ def sync(): log.error(e) actions.clear() - log.info("完成同步消费数据...") + log.info("完成同步相对价格排名数据...") diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index 9c660d8..151cbd0 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -28,7 +28,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') with ThreadPoolExecutor(50) as pool: def deal(index, item): diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 7f734cb..280bc58 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -28,7 +28,7 @@ def sync(city=None): seaweed_query = {'cat': 'district', 'status': 0} if pd.notna(city): seaweed_query['city'] = city - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') # 加载库中均价数据 query_sql = "SELECT city, region, name, avg_price FROM community_summary " diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index 5e08242..bc033d1 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -28,7 +28,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city cities = [city] - seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') + seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') deal_price_df = commutils.get_deal_avg_price(cities, 3) deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 3d87720..728bbaa 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -76,7 +76,7 @@ def sync(city=None): if pd.notna(city): seaweed_query['city'] = city - seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', + seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8') actions = list()