diff --git a/dags/business/business/community/consumption_stat.py b/dags/business/business/community/consumption_stat.py new file mode 100644 index 0000000..066a313 --- /dev/null +++ b/dags/business/business/community/consumption_stat.py @@ -0,0 +1,52 @@ +""" +财力评级统计 +""" +from goojia_utils import ElasticSearchClient, dateutils, nextcloudutils, dingutils +import pandas as pd +import numpy as np +import logging +import os +from elasticsearch import helpers +import matplotlib.pyplot as plt +import sys + +from sklearn.mixture import GaussianMixture +from sklearn.preprocessing import QuantileTransformer + +log = logging.getLogger(__name__) + +es206_client = ElasticSearchClient(host='192.168.0.89', username='elastic', password='elasticsearch') +index_name = "community_summary" + +save_path_prefix = "/home/airflow/data/community" + +save_png_path = save_path_prefix + "/consumption_rating" +if not os.path.exists(save_png_path): + os.makedirs(save_png_path) +nextcloud_aim_file_path = '财力评级/财力评级/财力评级统计报表' + +def stat(city): + """ + 计算消费潜力 + """ + sql = "select id, city, region, name, total_price, estate_charge, estate_charge_fix, relative_price_ranking, avg_cost, consumption_rating, price_rating, consumption_score from community_summary where city='" + city + "'" + community_summary_df = es206_client.scan_by_sql_to_data_frame(sql) + community_summary_df['diff'] = np.abs(community_summary_df['consumption_rating'] - community_summary_df['price_rating']) + maybe_wrong_community = community_summary_df[community_summary_df['diff'] > 2] + maybe_wrong_community = maybe_wrong_community.rename(columns={'id': '小区ID', 'city': '城市', 'region': '区域', + 'name': '小区名称', 'total_price': '户均总价', 'estate_charge': '物业费', 'estate_charge_fix': '物业费(修正后)', + 'relative_price_ranking': '相对价格排名', 'avg_cost': '人均消费', 'consumption_rating': '财力评级', 'price_rating': '价格评级', + 'consumption_score': '财力分数', 'diff': '差值'}) + aim_file = dateutils.format_date(dateutils.localize()) + '财力评级结果.csv' + maybe_wrong_community.to_csv(save_png_path + '/' + aim_file, index=False, columns=["城市", "区域", "小区名称", "财力评级", "价格评级", "财力分数", "差值", "户均总价", "相对价格排名", "人均消费", "物业费", "物业费(修正后)"]) + nextcloudutils.up_to_nextcloud(save_png_path + "/" + aim_file, nextcloud_aim_file_path + "/" + aim_file) + + dingutils.send_text("【统计】财力评级统计完毕,请点击 https://nextcloud.fangjia.com/s/nMbzk6ZKJDimapf 查看", "32400b949005817550937f82a82c064fe5be65c23204306442c15e8b485d7ef2") + +sys_city = sys.argv[1] if len(sys.argv) > 1 else None +if pd.notna(sys_city) and "," in sys_city: + cities = sys_city.split(",") + for city in cities: + stat(city) +else: + stat(sys_city) \ No newline at end of file diff --git a/dags/business/dag_community_summary.py b/dags/business/dag_community_summary.py index d3216fc..95b2bfd 100644 --- a/dags/business/dag_community_summary.py +++ b/dags/business/dag_community_summary.py @@ -94,6 +94,10 @@ with DAG( task_id='price_rating', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/price_rating.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) + consumption_stat = BashOperator( + task_id='consumption_stat', + bash_command='python /opt/airflow/dags/repo/dags/business/business/community/consumption_stat.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', + ) percentile = BashOperator( task_id='percentile', bash_command='python /opt/airflow/dags/repo/dags/business/business/community/percentile.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', @@ -120,5 +124,6 @@ with DAG( bash_command='python /opt/airflow/dags/repo/dags/business/business/community/full_name.py {{dag_run.conf["city"] if "city" in dag_run.conf}}', ) # 人均消费并行,值为空 - del_comm >> comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \ + del_comm >> comm >> price >> [avg_cost, total_price, relative_price_ranking, estate_charge_complete] >> total_price_complete >> consumption >> city_level_and_price_rating >> credit >> [estate_charge_ranking, price_rating, person_rating, percentile] >> \ + consumption_stat >> sell >> trade >> [traffic, business_district, medical, life, education] >> education2 >> extra >> other \ >> full_name