diff --git a/dags/jh/para/估值模型测试城市模板.xlsx b/dags/jh/para/估值模型测试城市模板.xlsx new file mode 100644 index 0000000..9318a3c Binary files /dev/null and b/dags/jh/para/估值模型测试城市模板.xlsx differ diff --git a/dags/jh/price/price_evaluation.py b/dags/jh/price/price_evaluation.py new file mode 100644 index 0000000..b8bb9cb --- /dev/null +++ b/dags/jh/price/price_evaluation.py @@ -0,0 +1,121 @@ +import os, sys +import logging +import pandas as pd +import requests +from datetime import datetime +from goojia_utils import dateutils, oracle + +os.environ["NLS_LANG"] = "American_America.AL32UTF8" +os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" + + +important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"] +city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") + + +def convert_calc_type(cale_type): + if cale_type == 1: + return "指导价" + elif cale_type == 2: + return "样本" + elif cale_type == 3: + return "神经网络" + + +fitment_code_map = {"0601": "毛坯", "0602": "简装", "0603": "精装"} + + +def convert_fitment_code(fitment_code): + if pd.isna(fitment_code): + return None + fitment_code = str(fitment_code) + if fitment_code in fitment_code_map: + return fitment_code_map[fitment_code] + return None + + +house_aspect_code_map = {"0301": "南北", "0302": "东南", "0303": "南", "0304": "东西", "0305": "东", "0306": "西南", "0307": "西", + "0308": "北", "0309": "东北", + "0310": "西北"} + + +def convert_house_aspect_code(house_aspect_code): + if pd.isna(house_aspect_code): + return None + house_aspect_code = str(house_aspect_code) + if house_aspect_code in house_aspect_code_map: + return house_aspect_code_map[house_aspect_code] + return None + +log = logging.getLogger(__name__) +path = "/home/airflow/data/jh/price" +if not os.path.exists(path): + os.makedirs(path) +year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m")) +log.info(f"测试月份:{year_month}") +avm_evaluation = oracle.scan_by_sql_to_data_frame(f"SELECT * FROM avm_evaluation_gj where year_month={year_month}") +avm_evaluation_item = oracle.scan_by_sql_to_data_frame(f"SELECT * FROM avm_evaluation_item_gj where year_month={year_month}") +community_base_price = oracle.scan_by_sql_to_data_frame(f"SELECT commid,cityname,distname,commname, avg_price as comm_avg_price,confidence,calc_type from community_base_price_gj where year_month={year_month}") +communities = oracle.scan_by_sql_to_data_frame(f"select COMMID from bigdata_community_gj") +community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])] +avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code) +avm_evaluation_item["houseaspect"] = avm_evaluation_item["houseaspect"].map(convert_house_aspect_code) +avm_evaluation_item["calc_type"] = avm_evaluation_item["calc_type"].map(convert_calc_type) +community_base_price["calc_type"] = community_base_price["calc_type"].map(convert_calc_type) +avm_evaluation_item["dealdate"] = avm_evaluation_item["dealdate"].apply(lambda x : x.strftime("%Y-%m-%d") if pd.notna(x) else None) +avm_evaluation_item_third = avm_evaluation_item[avm_evaluation_item.data_type=='第三方'] +avm_evaluation_item_deal = avm_evaluation_item[avm_evaluation_item.data_type=='成交'] +import_avm_evaluation_item_third = avm_evaluation_item_third[avm_evaluation_item_third.cityname.isin(important_cities)] +import_avm_evaluation_item_deal = avm_evaluation_item_deal[avm_evaluation_item_deal.cityname.isin(important_cities)] +import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["target_avg_price"] +import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["deal_total_price"].apply(lambda x : int(x)) +import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["calc_avg_price"] +import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["total_price"].apply(lambda x : int(x)) +import_community_base_price = community_base_price[community_base_price.cityname.isin(important_cities)] +normal_community_base_price = community_base_price[~(community_base_price.cityname.isin(important_cities))] +import_avm_evaluation = avm_evaluation[avm_evaluation.cityname.isin(important_cities)] +avm_evaluation = avm_evaluation.merge(city_order, on="cityname", how="left") +import_community_base_price = import_community_base_price.merge(city_order, on="cityname", how="left") +normal_community_base_price = normal_community_base_price.merge(city_order, on="cityname", how="left") +import_avm_evaluation_item_deal = import_avm_evaluation_item_deal.merge(city_order, on="cityname", how="left") +import_avm_evaluation_item_third = import_avm_evaluation_item_third.merge(city_order, on="cityname", how="left") +avm_evaluation.sort_values(by="cityrank", inplace=True) +import_community_base_price.sort_values(by="cityrank", inplace=True) +normal_community_base_price.sort_values(by="cityrank", inplace=True) +import_avm_evaluation_item_deal.sort_values(by="cityrank", inplace=True) +import_avm_evaluation_item_third.sort_values(by="cityrank", inplace=True) + +file_name = f"{year_month}估值模型测试.xlsx" +aim_file_name = path + '/' + file_name +with pd.ExcelWriter(aim_file_name) as writer: + avm_evaluation.to_excel(writer, sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False) + import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False) + import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False) + import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False) +log.info("估值结果测试结束") +def up_to_nextcloud(): + """ + 上传统计信息至nextcloud + """ + from nextcloud import NextCloud + NEXTCLOUD_URL = "https://nextcloud.fangjia.com" + NEXTCLOUD_USERNAME = "shenjian@fangjia.com" + NEXTCLOUD_PASSWORD = "goojia123456" + nextcloud_file_name = '共享/建行/' + file_name + nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD) + nxc.upload_file(aim_file_name, nextcloud_file_name) + +def send_msg(): + infos = "【准确度测评】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情" + data = { + "msgtype": "text", + "text": { + "content": infos + } + } + # 发送消息至群 + requests.post( + url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736', + json=data) +up_to_nextcloud() +send_msg() \ No newline at end of file diff --git a/dags/jh/price_exception_dag.py b/dags/jh/price_exception_dag.py index 9ac94ac..5f2f3af 100644 --- a/dags/jh/price_exception_dag.py +++ b/dags/jh/price_exception_dag.py @@ -11,7 +11,13 @@ with DAG( description="建行每月价格异常检测脚本,默认检测本月与上月差异, 若对比上月与前月价格,设置type=\"202210\"(前月日期即可) 若输出样本存在与否,设置type=sample" ) as dag: t1 = BashOperator( + task_id='price_evaluation', + bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_evaluation.py {{dag_run.conf["year_month"] if "year_month" in dag_run.conf}}', + ) + t2 = BashOperator( task_id='price_exception', bash_command='python /opt/airflow/dags/repo/dags/jh/price/price_exception.py {{dag_run.conf["type"] if "type" in dag_run.conf}}', ) - t1 + + + t1 >> t2