户均总价计算逻辑调整

This commit is contained in:
shenjian@fangjia.com
2023-03-03 16:52:22 +08:00
parent 87e931f200
commit 4d6c59d043
2 changed files with 14 additions and 54 deletions
@@ -3,10 +3,8 @@
"""
import logging
import pandas as pd
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
from elasticsearch import helpers
from concurrent.futures import ThreadPoolExecutor
import re
import sys
log = logging.getLogger(__name__)
@@ -26,59 +24,20 @@ def build_doc(item):
return doc_dict
# def avg_price_map_deal(x):
# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price']
# return x
def deal(seaweed_df, index, item):
if index % 1000 == 0:
log.info("已处理" + str(index))
curr_city = item['city']
curr_region = item['region']
curr_name = item['name']
names = [curr_name]
if pd.notna(item['alias']) and '[]' != item['alias']:
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
if len(alias_list) > 0:
for alias_item in alias_list:
names.append(alias_item)
# 户均总价
# if (curr_city + curr_region + curr_name) in avg_price_map:
# avg_price = avg_price_map[curr_city + curr_region + curr_name]
# if pd.notna(avg_price):
# 由于覆盖率较低,暂不过滤
# min_avg_price = 0.8 * avg_price
# max_avg_rice = 1.2 * avg_price
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366)
seaweed_df.loc[index, 'total_price'] = total_price
def sync(city=None):
log.info(f"开始同步户均总价信息至206 {index_name}")
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
if pd.notna(city):
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
seaweed_df = seaweed_df.reset_index(drop=True)
# 加载库中均价数据
# query_sql = "SELECT city, region, name, avg_price FROM community_summary "
# if pd.notna(city):
# query_sql += " WHERE city='" + city + "' "
# community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
# community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1)
with ThreadPoolExecutor(30) as pool:
for index, item in seaweed_df.iterrows():
pool.submit(deal, seaweed_df, index, item)
query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 "
if pd.notna(city):
query_sql += " AND city='" + city + "' "
community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
actions = list()
count = 0
for index, item in seaweed_df.iterrows():
community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area']
community_summary_df['total_price'] = community_summary_df['total_price'].astype(int)
for index, item in community_summary_df.iterrows():
doc_info = build_doc(item)
action = {
+6 -5
View File
@@ -34,10 +34,11 @@ with DAG(
task_id='total_price',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
total_price_complete = BashOperator(
task_id='total_price_complete',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
# 采用户均总价计算新方式,无需补充
# total_price_complete = BashOperator(
# task_id='total_price_complete',
# bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
# )
traffic = BashOperator(
task_id='traffic',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
@@ -90,5 +91,5 @@ with DAG(
task_id='other',
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
)
comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
comm >> price >> avg_cost >> total_price >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other