diff --git a/dags/business/business/community/avg_cost.py b/dags/business/business/community/avg_cost.py index b9467d9..5731179 100644 --- a/dags/business/business/community/avg_cost.py +++ b/dags/business/business/community/avg_cost.py @@ -84,6 +84,7 @@ def update_status(new_df): helpers.bulk(es206_client.elastic_client, actions) actions.clear() + def sync(): log.info("开始同步消费数据...") df = es206_client.scan_by_sql_to_data_frame( @@ -145,3 +146,5 @@ def sync(): log.info("完成同步消费数据...") + +sync() \ No newline at end of file diff --git a/dags/business/business/community/comm.py b/dags/business/business/community/comm.py index c5d61f5..1aab31b 100644 --- a/dags/business/business/community/comm.py +++ b/dags/business/business/community/comm.py @@ -19,6 +19,7 @@ path = "/home/airflow/data/community" if not os.path.exists(path): os.makedirs(path) + def build_doc(item): """ 构建字典 @@ -64,9 +65,9 @@ def sync(city=None): seaweed_df = pd.DataFrame( list(seaweed_conn.find(seaweed_query, {'city': 1, 'region': 1, 'name': 1, 'lng': 1, 'lat': 1, - 'alias': 1, 'estate_type2': 1, 'build_type': 1, - 'household_count': 1, - 'estate_charge': 1, 'finish_building_date': 1}) + 'alias': 1, 'estate_type2': 1, 'build_type': 1, + 'household_count': 1, + 'estate_charge': 1, 'finish_building_date': 1}) .batch_size(100000))) community_summary_df = es206_client.scan_by_sql_to_data_frame(f"select id from {index_name}") @@ -126,3 +127,6 @@ def sync(city=None): # 写入csv文件,防止小区变动导致的后续过程更新报错 seaweed_df.to_csv(path + '/seaweed.csv', sep='@', encoding='UTF-8', index=False) log.info(f"完成同步小区基本属性至206 {index_name}") + + +sync() diff --git a/dags/business/business/community/community_summary.py b/dags/business/business/community/community_summary.py deleted file mode 100644 index 95b17d8..0000000 --- a/dags/business/business/community/community_summary.py +++ /dev/null @@ -1,32 +0,0 @@ -import logging -import sys -import comm -import price -import sell -import trade -import avg_cost -import total_price -import traffic -import medical -import life -import relative_price_ranking -import consumption_potential - -log = logging.getLogger(__name__) - -type = sys.argv[1] if len(sys.argv) > 1 else 'all' -log.info("同步类型:" + type) - - -if type == 'all': - comm.sync() - price.sync() - sell.sync() - avg_cost.sync() - trade.sync() - total_price.sync() - traffic.sync() - medical.sync() - life.sync() - relative_price_ranking.sync() - consumption_potential.sync() diff --git a/dags/business/business/community/consumption_potential.py b/dags/business/business/community/consumption_potential.py index 30f4e8e..03f5bd4 100644 --- a/dags/business/business/community/consumption_potential.py +++ b/dags/business/business/community/consumption_potential.py @@ -16,36 +16,50 @@ def calc_consumption_potential(city): if pd.notna(city): sql += " where city='" + city + "'" community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) - community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.mean) + community_summary_df['total_price_mean'] = community_summary_df.groupby(by=['city'])['total_price'].transform( + np.mean) community_summary_df['total_price_std'] = community_summary_df.groupby(by=['city'])['total_price'].transform(np.std) - community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.mean) - community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform(np.std) + community_summary_df['estate_charge_mean'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform( + np.mean) + community_summary_df['estate_charge_std'] = community_summary_df.groupby(by=['city'])['estate_charge'].transform( + np.std) - community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.mean) - community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])['relative_price_ranking'].transform(np.std) + community_summary_df['relative_price_ranking_mean'] = community_summary_df.groupby(by=['city'])[ + 'relative_price_ranking'].transform(np.mean) + community_summary_df['relative_price_ranking_std'] = community_summary_df.groupby(by=['city'])[ + 'relative_price_ranking'].transform(np.std) community_summary_df['avg_cost_mean'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.mean) community_summary_df['avg_cost_std'] = community_summary_df.groupby(by=['city'])['avg_cost'].transform(np.std) - community_summary_df['total_price_score'] = 286.0134196 * (community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df['total_price_std'] + community_summary_df['total_price_score'] = 286.0134196 * ( + community_summary_df['total_price'] - community_summary_df['total_price_mean']) / community_summary_df[ + 'total_price_std'] community_summary_df['total_price_score'].fillna(0, inplace=True) - community_summary_df['estate_charge_score'] = 139.1374524 * (community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / community_summary_df['estate_charge_std'] + community_summary_df['estate_charge_score'] = 139.1374524 * ( + community_summary_df['estate_charge'] - community_summary_df['estate_charge_mean']) / \ + community_summary_df['estate_charge_std'] community_summary_df['estate_charge_score'].fillna(0, inplace=True) - community_summary_df['relative_price_ranking_score'] = 21.8427926 * (community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / community_summary_df['relative_price_ranking_std'] + community_summary_df['relative_price_ranking_score'] = 21.8427926 * ( + community_summary_df['relative_price_ranking'] - community_summary_df['relative_price_ranking_mean']) / \ + community_summary_df['relative_price_ranking_std'] community_summary_df['relative_price_ranking_score'].fillna(0, inplace=True) - community_summary_df['avg_cost_score'] = 67.0195871700 * (community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df['avg_cost_std'] + community_summary_df['avg_cost_score'] = 67.0195871700 * ( + community_summary_df['avg_cost'] - community_summary_df['avg_cost_mean']) / community_summary_df[ + 'avg_cost_std'] community_summary_df['avg_cost_score'].fillna(0, inplace=True) - community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df['estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \ - + community_summary_df['avg_cost_score'] + community_summary_df['avg_cost_score'] + 4099.978248 + community_summary_df['consumption_potential'] = community_summary_df['total_price_score'] + community_summary_df[ + 'estate_charge_score'] + community_summary_df['relative_price_ranking_score'] \ + + community_summary_df['avg_cost_score'] + community_summary_df[ + 'avg_cost_score'] + 4099.978248 pass - def calc_consumption_rating(): """ 计算消费等级 @@ -55,4 +69,7 @@ def calc_consumption_rating(): def sync(city=None): calc_consumption_potential(city) - pass \ No newline at end of file + pass + + +sync() diff --git a/dags/business/business/community/life.py b/dags/business/business/community/life.py index 12ebde9..47bdab3 100644 --- a/dags/business/business/community/life.py +++ b/dags/business/business/community/life.py @@ -94,4 +94,7 @@ def sync(city=None): log.error(e) actions.clear() - log.info(f"完成同步生活配套至206 {index_name}") \ No newline at end of file + log.info(f"完成同步生活配套至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/business/community/medical.py b/dags/business/business/community/medical.py index bcb28a0..79d8bb4 100644 --- a/dags/business/business/community/medical.py +++ b/dags/business/business/community/medical.py @@ -82,4 +82,7 @@ def sync(city=None): log.error(e) actions.clear() - log.info(f"完成同步医院至206 {index_name}") \ No newline at end of file + log.info(f"完成同步医院至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/business/community/price.py b/dags/business/business/community/price.py index f5d306a..f121329 100644 --- a/dags/business/business/community/price.py +++ b/dags/business/business/community/price.py @@ -115,3 +115,6 @@ def sync(city=None): print(e) actions.clear() log.info(f"完成同步价格信息至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/business/community/relative_price_ranking.py b/dags/business/business/community/relative_price_ranking.py index e87605b..d6842a9 100644 --- a/dags/business/business/community/relative_price_ranking.py +++ b/dags/business/business/community/relative_price_ranking.py @@ -107,3 +107,5 @@ def sync(): log.info("完成同步相对价格排名数据...") + +sync() \ No newline at end of file diff --git a/dags/business/business/community/sell.py b/dags/business/business/community/sell.py index db92878..0b5c9d3 100644 --- a/dags/business/business/community/sell.py +++ b/dags/business/business/community/sell.py @@ -81,3 +81,6 @@ def sync(city=None): print(e) actions.clear() log.info(f"完成同步出售信息至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/business/community/total_price.py b/dags/business/business/community/total_price.py index 2b74fdc..0e6cbf7 100644 --- a/dags/business/business/community/total_price.py +++ b/dags/business/business/community/total_price.py @@ -23,6 +23,7 @@ def build_doc(item): doc_dict['total_price'] = int(item['total_price']) return doc_dict + def sync(city=None): log.info(f"开始同步户均总价信息至206 {index_name}") @@ -66,14 +67,13 @@ def sync(city=None): if pd.notna(avg_price): min_avg_price = 0.8 * avg_price max_avg_rice = 1.2 * avg_price - total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366, min_avg_price=min_avg_price, max_avg_price=max_avg_rice) + total_price = commutils.get_total_price(curr_city, curr_region, names, day_before=366, + min_avg_price=min_avg_price, max_avg_price=max_avg_rice) seaweed_df.loc[index, 'total_price'] = total_price for index, item in seaweed_df.iterrows(): pool.submit(deal, index, item) - - actions = list() count = 0 for index, item in seaweed_df.iterrows(): @@ -102,3 +102,6 @@ def sync(city=None): print(e) actions.clear() log.info(f"完成同步户均总价信息至206 {index_name}") + + +sync() diff --git a/dags/business/business/community/trade.py b/dags/business/business/community/trade.py index 85b9582..ad2aa5d 100644 --- a/dags/business/business/community/trade.py +++ b/dags/business/business/community/trade.py @@ -33,11 +33,11 @@ def sync(city=None): deal_price_df = commutils.get_deal_avg_price(cities, 3) deal_price_df = deal_price_df[['match_city', 'match_region', 'match_name', 'avg_price']] - deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name', 'avg_price': 'three_month_trade_avg_price'}) + deal_price_df = deal_price_df.rename(columns={'match_city': 'city', 'match_region': 'region', 'match_name': 'name', + 'avg_price': 'three_month_trade_avg_price'}) seaweed_df = pd.merge(seaweed_df, deal_price_df, how='left', on=['city', 'region', 'name']) - actions = list() count = 0 for index, item in seaweed_df.iterrows(): @@ -65,4 +65,7 @@ def sync(city=None): except Exception as e: print(e) actions.clear() - log.info(f"完成同步成交信息至206 {index_name}") \ No newline at end of file + log.info(f"完成同步成交信息至206 {index_name}") + + +sync() diff --git a/dags/business/business/community/traffic.py b/dags/business/business/community/traffic.py index 9cba530..7f7512e 100644 --- a/dags/business/business/community/traffic.py +++ b/dags/business/business/community/traffic.py @@ -91,4 +91,7 @@ def sync(city=None): print(e) actions.clear() - log.info(f"完成同步交通至206 {index_name}") \ No newline at end of file + log.info(f"完成同步交通至206 {index_name}") + + +sync() \ No newline at end of file diff --git a/dags/business/community_summary_dag.py b/dags/business/community_summary_dag.py index c1ee918..65b53b3 100644 --- a/dags/business/community_summary_dag.py +++ b/dags/business/community_summary_dag.py @@ -11,7 +11,48 @@ with DAG( description="地址输出信息分类数据定时更新" ) as dag: t1 = BashOperator( - task_id='community_summary_dag', - bash_command='python /opt/airflow/dags/repo/dags/business/business/community/community_summary.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + task_id='comm_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/comm.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - t1 \ No newline at end of file + t2 = BashOperator( + task_id='price_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t3 = BashOperator( + task_id='sell_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/sell.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t4 = BashOperator( + task_id='avg_cost_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/avg_cost.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t5 = BashOperator( + task_id='trade_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/trade.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t6 = BashOperator( + task_id='total_price_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/total_price.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t7 = BashOperator( + task_id='traffic_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/traffic.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t8 = BashOperator( + task_id='medical_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/medical.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t9 = BashOperator( + task_id='life_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/life.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t10 = BashOperator( + task_id='relative_price_ranking_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/relative_price_ranking.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + t11 = BashOperator( + task_id='consumption_potential_dag', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_potential.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', + ) + + t1 >> t2 >> t3 >> t4 >> t5 >> t6 >> t7 >> t8 >> t9 >> t10 >> t11 \ No newline at end of file