From a2cb88651fb029dc3fb05cd0ccb80651f74b2e2e Mon Sep 17 00:00:00 2001 From: xuhe Date: Fri, 24 Feb 2023 18:25:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=97=A0=E4=BB=B7=E6=A0=BC?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E6=95=B0=E6=8D=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/price/no_price_community.py | 63 ++++++++++++++++++++++++ dags/jh/price/price_deviation.py | 74 +++++++++++++++++++++++++++++ dags/jh/price_exception_dag.py | 11 ++++- 3 files changed, 146 insertions(+), 2 deletions(-) create mode 100644 dags/jh/price/no_price_community.py create mode 100644 dags/jh/price/price_deviation.py diff --git a/dags/jh/price/no_price_community.py b/dags/jh/price/no_price_community.py new file mode 100644 index 0000000..49db8d7 --- /dev/null +++ b/dags/jh/price/no_price_community.py @@ -0,0 +1,63 @@ +import os, sys +import logging +import pandas as pd +import requests +from datetime import datetime +from goojia_utils import dateutils, oracle +from sqlalchemy import create_engine + +os.environ["NLS_LANG"] = "American_America.AL32UTF8" +os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" +engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) + +log = logging.getLogger(__name__) +path = "/home/airflow/data/jh/price" +if not os.path.exists(path): + os.makedirs(path) +year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m")) +log.info(f"测试月份:{year_month}") + +avm_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ", con=engine) +comm_df = pd.read_sql(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ", con=engine) +price_df = pd.read_sql(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ", con=engine) +comm_df = comm_df.merge(avm_df, on="urbn_id", how="left") +comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left") +no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']] +avm_df.loc[:,'无均价数量'] = avm_df['gujia_bds_comm_num'] - avm_df['gujiabdsvalmnycommnum'] +avm_df.loc[:,'num'] = 60 - avm_df['gujiabdsvalmnycommnum'] +avm_df = avm_df[avm_df['无均价数量'] > 0] +avm_df = avm_df[['urbn_id','urbn_nm','num']] + +file_name = f"{year_month}无价格小区统计数据.xlsx" +aim_file_name = path + '/' + file_name +with pd.ExcelWriter(aim_file_name) as writer: + no_price_comm_df.to_excel(writer, sheet_name="无价格小区列表", columns=['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm'], header=["城市id", "城市名称","区域","小区id","小区名称"], index=False) + avm_df.to_excel(writer, sheet_name="待补充数量", columns=['urbn_id','urbn_nm','num'], header=["城市id","城市名称","待补充数量"],index=False) + +log.info("无价格小区统计数据结束") +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(aim_file_name, nextcloud_file_name) + +def send_msg(): + infos = "【准确度测评】无价格小区统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情" + data = { + "msgtype": "text", + "text": { + "content": infos + } + } + # 发送消息至群 + requests.post( + url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data) +up_to_nextcloud() +send_msg() \ No newline at end of file diff --git a/dags/jh/price/price_deviation.py b/dags/jh/price/price_deviation.py new file mode 100644 index 0000000..53ea7f4 --- /dev/null +++ b/dags/jh/price/price_deviation.py @@ -0,0 +1,74 @@ +import os, sys +import logging +import pandas as pd +import requests +from datetime import datetime +from goojia_utils import dateutils, oracle +from sqlalchemy import create_engine + +os.environ["NLS_LANG"] = "American_America.AL32UTF8" +os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" +engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) + +log = logging.getLogger(__name__) +path = "/home/airflow/data/jh/price" +if not os.path.exists(path): + os.makedirs(path) +month_now = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now()), '%Y%m') +month_last = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-1), '%Y%m') +month_before = dateutils.format_date(dateutils.add_months(dateutils.datetime.datetime.now(), months=-2), '%Y%m') + +type = sys.argv[1] if len(sys.argv) > 1 else '' +log.info("输出类型" + type) + +# 新增支持上一月与前月对比 +if type == month_last: + month_now = month_last + month_last = month_before +log.info(f"测试月份:{month_now}") + +price_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ", con=engine) +deal_df = pd.read_sql(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ", con=engine) +deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left") +deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4) +deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15] + +# third_df = pd.read_sql(f"select gujia_trdpt_avprc_id,gujia_bds_id as gujia_comm_id,gujia_dtsr_dsc,gujia_bds_comm_avprc from base.t0831_bigdata_third_party_price_gj_h where gujiabds_avprc_dt_val={month_last} ", con=engine) +# third_price_df = third_df.merge(price_df, on="gujia_comm_id", how="left") +# third_price_df["ppe"] = round((third_price_df["gujia_comm_basic_prc"] - third_price_df['gujia_bds_comm_avprc']) / third_price_df['gujia_bds_comm_avprc'], 4) +# third_price_df = third_price_df[abs(third_price_df['ppe']) > 0.15] + + +file_name = f"{month_now}测试偏差较大案例统计数据.xlsx" +aim_file_name = path + '/' + file_name +with pd.ExcelWriter(aim_file_name) as writer: + deal_price_df.to_excel(writer, sheet_name="成交", columns=['urbn_id','urbn_nm','gujia_bds_mdl_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_trdtckt_prc','PPE'], header=["城市id", "城市名称","成交id","区域","小区id","小区名称",'计算类型','计算价格','成交价格','PPE'], index=False) + # third_price_df.to_excel(writer, sheet_name="第三方", columns=['urbn_id','urbn_nm','gujia_trdpt_avprc_id','gujia_admndiv_nm','gujia_comm_id','comm_nm','gujia_bds_clmtd','gujia_comm_basic_prc','gujia_bds_comm_avprc','PPE','gujia_dtsr_dsc'], header=["城市id", "城市名称","第三方id","区域","小区id","小区名称",'计算类型','计算价格','第三方价格','PPE','来源'],index=False) + +log.info("偏差较大案例统计数据结束") +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(aim_file_name, nextcloud_file_name) + +def send_msg(): + infos = "【准确度测评】测试偏差较大案例统计数据生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情" + data = { + "msgtype": "text", + "text": { + "content": infos + } + } + # 发送消息至群 + requests.post( + url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data) +up_to_nextcloud() +send_msg() \ No newline at end of file diff --git a/dags/jh/price_exception_dag.py b/dags/jh/price_exception_dag.py index 5e22e0c..68664ee 100644 --- a/dags/jh/price_exception_dag.py +++ b/dags/jh/price_exception_dag.py @@ -18,6 +18,13 @@ with DAG( task_id='price_exception', bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', ) + t3 = BashOperator( + task_id='no_price_community', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/no_price_community.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', + ) + t4 = BashOperator( + task_id='price_deviation', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_deviation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', + ) - - t1 >> t2 + t1 >> t2 >> t3 >> t4