户均总价计算逻辑调整
This commit is contained in:
@@ -3,10 +3,8 @@
|
||||
"""
|
||||
import logging
|
||||
import pandas as pd
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils
|
||||
from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils
|
||||
from elasticsearch import helpers
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import re
|
||||
import sys
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
@@ -26,59 +24,20 @@ def build_doc(item):
|
||||
return doc_dict
|
||||
|
||||
|
||||
# def avg_price_map_deal(x):
|
||||
# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price']
|
||||
# return x
|
||||
|
||||
|
||||
def deal(seaweed_df, index, item):
|
||||
if index % 1000 == 0:
|
||||
log.info("已处理" + str(index))
|
||||
curr_city = item['city']
|
||||
curr_region = item['region']
|
||||
curr_name = item['name']
|
||||
names = [curr_name]
|
||||
|
||||
if pd.notna(item['alias']) and '[]' != item['alias']:
|
||||
alias_list = re.sub("\[|\]|'", '', item['alias']).split(',')
|
||||
if len(alias_list) > 0:
|
||||
for alias_item in alias_list:
|
||||
names.append(alias_item)
|
||||
|
||||
# 户均总价
|
||||
# if (curr_city + curr_region + curr_name) in avg_price_map:
|
||||
# avg_price = avg_price_map[curr_city + curr_region + curr_name]
|
||||
# if pd.notna(avg_price):
|
||||
# 由于覆盖率较低,暂不过滤
|
||||
# min_avg_price = 0.8 * avg_price
|
||||
# max_avg_rice = 1.2 * avg_price
|
||||
total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366)
|
||||
seaweed_df.loc[index, 'total_price'] = total_price
|
||||
|
||||
|
||||
def sync(city=None):
|
||||
log.info(f"开始同步户均总价信息至206 {index_name}")
|
||||
|
||||
seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8')
|
||||
if pd.notna(city):
|
||||
seaweed_df = seaweed_df[seaweed_df['city'] == city].copy()
|
||||
seaweed_df = seaweed_df.reset_index(drop=True)
|
||||
|
||||
# 加载库中均价数据
|
||||
# query_sql = "SELECT city, region, name, avg_price FROM community_summary "
|
||||
# if pd.notna(city):
|
||||
# query_sql += " WHERE city='" + city + "' "
|
||||
# community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
||||
|
||||
# community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1)
|
||||
|
||||
with ThreadPoolExecutor(30) as pool:
|
||||
for index, item in seaweed_df.iterrows():
|
||||
pool.submit(deal, seaweed_df, index, item)
|
||||
query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 "
|
||||
if pd.notna(city):
|
||||
query_sql += " AND city='" + city + "' "
|
||||
community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql)
|
||||
|
||||
actions = list()
|
||||
count = 0
|
||||
for index, item in seaweed_df.iterrows():
|
||||
community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area']
|
||||
community_summary_df['total_price'] = community_summary_df['total_price'].astype(int)
|
||||
for index, item in community_summary_df.iterrows():
|
||||
doc_info = build_doc(item)
|
||||
|
||||
action = {
|
||||
|
||||
@@ -34,10 +34,11 @@ with DAG(
|
||||
task_id='total_price',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
total_price_complete = BashOperator(
|
||||
task_id='total_price_complete',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
# 采用户均总价计算新方式,无需补充
|
||||
# total_price_complete = BashOperator(
|
||||
# task_id='total_price_complete',
|
||||
# bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
# )
|
||||
traffic = BashOperator(
|
||||
task_id='traffic',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
@@ -90,5 +91,5 @@ with DAG(
|
||||
task_id='other',
|
||||
bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}',
|
||||
)
|
||||
comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
|
||||
comm >> price >> avg_cost >> total_price >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \
|
||||
>> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other
|
||||
|
||||
Reference in New Issue
Block a user