新增相对价格排名
This commit is contained in:
@@ -100,7 +100,7 @@ def sync():
|
||||
# 按最新的百分位更新库中价格状态
|
||||
update_status(new_df)
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
|
||||
@@ -94,7 +94,7 @@ def sync(city=None):
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print("更新:" + str(count))
|
||||
log.info("更新:" + str(count))
|
||||
actions.clear()
|
||||
|
||||
# 对于不存在的插入
|
||||
@@ -113,12 +113,12 @@ def sync(city=None):
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
count += len(actions)
|
||||
print("插入:" + str(count))
|
||||
log.info("插入:" + str(count))
|
||||
actions.clear()
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
# 写入csv文件,防止小区变动导致的后续过程更新报错
|
||||
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
seaweed_df.to_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8', index=False)
|
||||
log.info(f"完成同步小区基本属性至206 {index_name}")
|
||||
|
||||
@@ -9,6 +9,7 @@ import total_price
|
||||
import traffic
|
||||
import medical
|
||||
import life
|
||||
import relative_price_ranking
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
@@ -26,3 +27,4 @@ if type == 'all':
|
||||
traffic.sync()
|
||||
medical.sync()
|
||||
life.sync()
|
||||
relative_price_ranking.sync()
|
||||
|
||||
@@ -79,7 +79,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -67,7 +67,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
with ThreadPoolExecutor(max_workers=30) as pool:
|
||||
|
||||
@@ -42,7 +42,7 @@ def sync(city=None):
|
||||
cities = [city]
|
||||
else:
|
||||
cities = None
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
# 小区所在区域均价
|
||||
region_price_df = commutils.get_region_price(city=city)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
"""
|
||||
相对价格排名计算同步至206
|
||||
即周边一公里内小区价格排名0-1之间
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
@@ -21,10 +22,18 @@ def find_around_seaweed(lng, lat):
|
||||
"""
|
||||
if pd.isna(lng):
|
||||
return None
|
||||
ret_es = es206_client.elastic_client.search({
|
||||
ret_es = es206_client.scan_to_data_frame('community_summary', {
|
||||
"_source": ["id", "avg_price"],
|
||||
"query": {
|
||||
"bool": {
|
||||
"filter": [
|
||||
{
|
||||
"range": {
|
||||
"avg_price": {
|
||||
"gt": 0
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"geo_distance": {
|
||||
"distance": 1000,
|
||||
@@ -34,64 +43,31 @@ def find_around_seaweed(lng, lat):
|
||||
]
|
||||
}
|
||||
}
|
||||
}, index='community_summary')
|
||||
return ret_es['aggregations']['price']['value']
|
||||
})
|
||||
return ret_es
|
||||
|
||||
|
||||
def build_doc(item):
|
||||
doc_dict = dict()
|
||||
if 'avg_cost' in item and pd.notna(item['avg_cost']):
|
||||
doc_dict['avg_cost'] = item['avg_cost']
|
||||
if 'relative_price_ranking' in item and pd.notna(item['relative_price_ranking']):
|
||||
doc_dict['relative_price_ranking'] = item['relative_price_ranking']
|
||||
return doc_dict
|
||||
|
||||
|
||||
def update_status(new_df):
|
||||
actions = list()
|
||||
|
||||
def deal(item):
|
||||
action_doc = {
|
||||
"_op_type": "update",
|
||||
"_index": "bd_poi",
|
||||
"_id": item['_id'],
|
||||
"doc": {'status': 0}
|
||||
}
|
||||
actions.append(action_doc)
|
||||
if len(actions) == 5000:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
return item
|
||||
|
||||
new_df.apply(lambda x: deal(x), axis=1)
|
||||
if len(actions) > 0:
|
||||
helpers.bulk(es206_client.elastic_client, actions)
|
||||
actions.clear()
|
||||
|
||||
def sync():
|
||||
log.info("开始同步消费数据...")
|
||||
df = es206_client.scan_by_sql_to_data_frame(
|
||||
"select id, city, tag, type, price from bd_poi where price > 0 and tag in('美食;中餐厅', '美食;外国餐厅')")
|
||||
log.info("开始同步相对价格排名数据...")
|
||||
|
||||
# 按百分位5-95筛选数据
|
||||
cities = df['city'].unique()
|
||||
for city in cities:
|
||||
df.loc[df['city'] == city, 'minPrice5'] = np.percentile(a=df[df['city'] == city]['price'], q=5, axis=0)
|
||||
df.loc[df['city'] == city, 'minPrice95'] = np.percentile(a=df[df['city'] == city]['price'], q=95, axis=0)
|
||||
|
||||
new_df = df[(df['price'] >= df['minPrice5']) & (df['price'] <= df['minPrice95'])]
|
||||
new_df['new_mean'] = new_df.groupby(by=['city'])['price'].transform(np.mean)
|
||||
|
||||
# 按最新的百分位更新库中价格状态
|
||||
update_status(new_df)
|
||||
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
def find_around_price_item(index, item):
|
||||
if 'lng' not in item:
|
||||
return
|
||||
lng = commutils.format_lat_lng(item['lng'])
|
||||
lat = commutils.format_lat_lng(item['lat'])
|
||||
price = find_around_price(lng, lat)
|
||||
seaweed_df.loc[index, 'avg_cost'] = price
|
||||
if pd.notna(lng):
|
||||
around_seaweed_df = find_around_seaweed(lng, lat)
|
||||
around_seaweed_df['relative_price_ranking'] = around_seaweed_df['avg_price'].rank(pct=True)
|
||||
seaweed_df.loc[index, 'relative_price_ranking'] = around_seaweed_df.loc[around_seaweed_df['id'] == item['_id']]['relative_price_ranking'].tolist()[0]
|
||||
|
||||
with ThreadPoolExecutor(max_workers=50) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
@@ -125,5 +101,5 @@ def sync():
|
||||
log.error(e)
|
||||
actions.clear()
|
||||
|
||||
log.info("完成同步消费数据...")
|
||||
log.info("完成同步相对价格排名数据...")
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ def sync(city=None):
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
with ThreadPoolExecutor(50) as pool:
|
||||
def deal(index, item):
|
||||
|
||||
@@ -28,7 +28,7 @@ def sync(city=None):
|
||||
seaweed_query = {'cat': 'district', 'status': 0}
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
# 加载库中均价数据
|
||||
query_sql = "SELECT city, region, name, avg_price FROM community_summary "
|
||||
|
||||
@@ -28,7 +28,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
cities = [city]
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
|
||||
deal_price_df = commutils.get_deal_avg_price(cities, 3)
|
||||
deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']]
|
||||
|
||||
@@ -76,7 +76,7 @@ def sync(city=None):
|
||||
if pd.notna(city):
|
||||
seaweed_query['city'] = city
|
||||
|
||||
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/community/data/seaweed.csv', sep='@',
|
||||
seaweed_df = seaweed_df = pd.read_csv('/opt/airflow/dags/repo/dags/business/business/community/data/seaweed.csv', sep='@',
|
||||
encoding='UTF-8')
|
||||
|
||||
actions = list()
|
||||
|
||||
Reference in New Issue
Block a user