diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 91082fb..254fec3 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -3,10 +3,8 @@ """ import logging import pandas as pd -from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, commutils, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, ElasticSearchClient, dateutils from elasticsearch import helpers -from concurrent.futures import ThreadPoolExecutor -import re import sys log = logging.getLogger(__name__) @@ -26,59 +24,20 @@ def build_doc(item): return doc_dict -# def avg_price_map_deal(x): -# avg_price_map[x['city'] + x['region'] + x['name']] = x['avg_price'] -# return x - - -def deal(seaweed_df, index, item): - if index % 1000 == 0: - log.info("已处理" + str(index)) - curr_city = item['city'] - curr_region = item['region'] - curr_name = item['name'] - names = [curr_name] - - if pd.notna(item['alias']) and '[]' != item['alias']: - alias_list = re.sub("\[|\]|'", '', item['alias']).split(',') - if len(alias_list) > 0: - for alias_item in alias_list: - names.append(alias_item) - - # 户均总价 - # if (curr_city + curr_region + curr_name) in avg_price_map: - # avg_price = avg_price_map[curr_city + curr_region + curr_name] - # if pd.notna(avg_price): - # 由于覆盖率较低,暂不过滤 - # min_avg_price = 0.8 * avg_price - # max_avg_rice = 1.2 * avg_price - total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366) - seaweed_df.loc[index, 'total_price'] = total_price - - def sync(city=None): log.info(f"开始同步户均总价信息至206 {index_name}") - seaweed_df = pd.read_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8') - if pd.notna(city): - seaweed_df = seaweed_df[seaweed_df['city'] == city].copy() - seaweed_df = seaweed_df.reset_index(drop=True) - # 加载库中均价数据 - # query_sql = "SELECT city, region, name, avg_price FROM community_summary " - # if pd.notna(city): - # query_sql += " WHERE city='" + city + "' " - # community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql) - - # community_summary_df.apply(lambda x: avg_price_map_deal(x), axis=1) - - with ThreadPoolExecutor(30) as pool: - for index, item in seaweed_df.iterrows(): - pool.submit(deal, seaweed_df, index, item) + query_sql = "SELECT city, region, name, avg_price, house_avg_area FROM community_summary WHERE avg_price > 0 and house_avg_area > 0 " + if pd.notna(city): + query_sql += " AND city='" + city + "' " + community_summary_df = es206_client.scan_by_sql_to_data_frame(query_sql) actions = list() count = 0 - for index, item in seaweed_df.iterrows(): + community_summary_df['total_price'] = community_summary_df['avg_price'] * community_summary_df['house_avg_area'] + community_summary_df['total_price'] = community_summary_df['total_price'].astype(int) + for index, item in community_summary_df.iterrows(): doc_info = build_doc(item) action = { diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index 94523c0..3125a13 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -34,10 +34,11 @@ with DAG( task_id='total_price', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - total_price_complete = BashOperator( - task_id='total_price_complete', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', - ) + # 采用户均总价计算新方式,无需补充 + # total_price_complete = BashOperator( + # task_id='total_price_complete', + # bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price_complete.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + # ) traffic = BashOperator( task_id='traffic', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', @@ -90,5 +91,5 @@ with DAG( task_id='other', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/other.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) - comm >> price >> avg_cost >> total_price >> total_price_complete >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \ + comm >> price >> avg_cost >> total_price >> relative_price_ranking >> estate_charge_complete >> estate_charge_ranking \ >> consumption >> price_rating >> sell >> trade >> traffic >> business_district >> medical >> life >> education >> education2 >> extra >> other