新增相对价格排名

This commit is contained in:
shenjian@fangjia.com
2022-10-24 18:18:53 +08:00
parent ee8d87b957
commit 21e5c5fdac
12 changed files with 34 additions and 56 deletions
+1 -1
View File
@@ -100,7 +100,7 @@ def sync():
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
def find_around_price_item(index, item):
if 'lng' not in item:
+3 -3
View File
@@ -94,7 +94,7 @@ def sync(city=None):
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
print("更新:" + str(count))
log.info("更新:" + str(count))
actions.clear()
# 对于不存在的插入
@@ -113,12 +113,12 @@ def sync(city=None):
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
count += len(actions)
print("插入:" + str(count))
log.info("插入:" + str(count))
actions.clear()
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
# 写入csv文件,防止小区变动导致的后续过程更新报错
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
log.info(f"完成同步小区基本属性至206 {index_name}")
@@ -9,6 +9,7 @@ import total_price
import traffic
import medical
import life
import relative_price_ranking
log = logging.getLogger(__name__)
@@ -26,3 +27,4 @@ if type == 'all':
traffic.sync()
medical.sync()
life.sync()
relative_price_ranking.sync()
+1 -1
View File
@@ -79,7 +79,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+1 -1
View File
@@ -67,7 +67,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
actions = list()
with ThreadPoolExecutor(max_workers=30) as pool:
+1 -1
View File
@@ -42,7 +42,7 @@ def sync(city=None):
cities = [city]
else:
cities = None
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
# 小区所在区域均价
region_price_df = commutils.get_region_price(city=city)
@@ -1,5 +1,6 @@
"""
相对价格排名计算同步至206
即周边一公里内小区价格排名0-1之间
"""
import logging
import pandas as pd
@@ -21,10 +22,18 @@ def find_around_seaweed(lng, lat):
"""
if pd.isna(lng):
return None
ret_es = es206_client.elastic_client.search({
ret_es = es206_client.scan_to_data_frame('community_summary', {
"_source": ["id", "avg_price"],
"query": {
"bool": {
"filter": [
{
"range": {
"avg_price": {
"gt": 0
}
}
},
{
"geo_distance": {
"distance": 1000,
@@ -34,64 +43,31 @@ def find_around_seaweed(lng, lat):
]
}
}
}, index='community_summary')
return ret_es['aggregations']['price']['value']
})
return ret_es
def build_doc(item):
doc_dict = dict()
if 'avg_cost' in item and pd.notna(item['avg_cost']):
doc_dict['avg_cost'] = item['avg_cost']
if 'relative_price_ranking' in item and pd.notna(item['relative_price_ranking']):
doc_dict['relative_price_ranking'] = item['relative_price_ranking']
return doc_dict
def update_status(new_df):
actions = list()
def deal(item):
action_doc = {
"_op_type": "update",
"_index": "bd_poi",
"_id": item['_id'],
"doc": {'status': 0}
}
actions.append(action_doc)
if len(actions) == 5000:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
return item
new_df.apply(lambda x: deal(x), axis=1)
if len(actions) > 0:
helpers.bulk(es206_client.elastic_client, actions)
actions.clear()
def sync():
log.info("开始同步消费数据...")
df = es206_client.scan_by_sql_to_data_frame(
"select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')")
log.info("开始同步相对价格排名数据...")
# 按百分位5-95筛选数据
cities = df['city'].unique()
for city in cities:
df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0)
df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0)
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
# 按最新的百分位更新库中价格状态
update_status(new_df)
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
def find_around_price_item(index, item):
if 'lng' not in item:
return
lng = commutils.format_lat_lng(item['lng'])
lat = commutils.format_lat_lng(item['lat'])
price = find_around_price(lng, lat)
seaweed_df.loc[index, 'avg_cost'] = price
if pd.notna(lng):
around_seaweed_df = find_around_seaweed(lng, lat)
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
with ThreadPoolExecutor(max_workers=50) as pool:
for index, item in seaweed_df.iterrows():
@@ -125,5 +101,5 @@ def sync():
log.error(e)
actions.clear()
log.info("完成同步消费数据...")
log.info("完成同步相对价格排名数据...")
+1 -1
View File
@@ -28,7 +28,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
with ThreadPoolExecutor(50) as pool:
def deal(index, item):
@@ -28,7 +28,7 @@ def sync(city=None):
seaweed_query = {'cat': 'district', 'status': 0}
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
# 加载库中均价数据
query_sql = "SELECT city, region, name, avg_price FROM community_summary "
+1 -1
View File
@@ -28,7 +28,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
cities = [city]
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
deal_price_df = commutils.get_deal_avg_price(cities, 3)
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
+1 -1
View File
@@ -76,7 +76,7 @@ def sync(city=None):
if pd.notna(city):
seaweed_query['city'] = city
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@',
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@',
encoding='UTF-8')
actions = list()