124 lines
10 KiB
Python
124 lines
10 KiB
Python
import os, sys
|
|
import logging
|
|
import pandas as pd
|
|
import requests
|
|
from datetime import datetime
|
|
from goojia_utils import dateutils, pgsql
|
|
|
|
important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
|
|
city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
|
|
|
|
|
|
def convert_calc_type(cale_type):
|
|
if cale_type == 1:
|
|
return "指导价"
|
|
elif cale_type == 2:
|
|
return "样本"
|
|
elif cale_type == 3:
|
|
return "神经网络"
|
|
|
|
|
|
fitment_code_map = {"0601": "毛坯", "0602": "简装", "0603": "精装"}
|
|
|
|
|
|
def convert_fitment_code(fitment_code):
|
|
if pd.isna(fitment_code):
|
|
return None
|
|
fitment_code = str(fitment_code)
|
|
if fitment_code in fitment_code_map:
|
|
return fitment_code_map[fitment_code]
|
|
return None
|
|
|
|
|
|
house_aspect_code_map = {"0301": "南北", "0302": "东南", "0303": "南", "0304": "东西", "0305": "东", "0306": "西南", "0307": "西",
|
|
"0308": "北", "0309": "东北",
|
|
"0310": "西北"}
|
|
|
|
|
|
def convert_house_aspect_code(house_aspect_code):
|
|
if pd.isna(house_aspect_code):
|
|
return None
|
|
house_aspect_code = str(house_aspect_code)
|
|
if house_aspect_code in house_aspect_code_map:
|
|
return house_aspect_code_map[house_aspect_code]
|
|
return None
|
|
|
|
log = logging.getLogger(__name__)
|
|
path = "/home/airflow/data/jh/price"
|
|
if not os.path.exists(path):
|
|
os.makedirs(path)
|
|
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
|
|
log.info(f"测试月份:{year_month}")
|
|
avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'}
|
|
avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'}
|
|
avm_evaluation = pgsql.scan_by_sql_to_data_frame(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}")
|
|
avm_evaluation_item = pgsql.scan_by_sql_to_data_frame(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}")
|
|
avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True)
|
|
avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True)
|
|
community_base_price = pgsql.scan_by_sql_to_data_frame(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname,
|
|
gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}'''
|
|
)
|
|
communities = pgsql.scan_by_sql_to_data_frame(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h")
|
|
|
|
community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])]
|
|
avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code)
|
|
avm_evaluation_item["houseaspect"] = avm_evaluation_item["houseaspect"].map(convert_house_aspect_code)
|
|
avm_evaluation_item["calc_type"] = avm_evaluation_item["calc_type"].map(convert_calc_type)
|
|
community_base_price["calc_type"] = community_base_price["calc_type"].map(convert_calc_type)
|
|
avm_evaluation_item["dealdate"] = avm_evaluation_item["dealdate"].apply(lambda x : x.strftime("%Y-%m-%d") if pd.notna(x) else None)
|
|
avm_evaluation_item_third = avm_evaluation_item[avm_evaluation_item.data_type=='第三方']
|
|
avm_evaluation_item_deal = avm_evaluation_item[avm_evaluation_item.data_type=='成交']
|
|
import_avm_evaluation_item_third = avm_evaluation_item_third[avm_evaluation_item_third.cityname.isin(important_cities)]
|
|
import_avm_evaluation_item_deal = avm_evaluation_item_deal[avm_evaluation_item_deal.cityname.isin(important_cities)]
|
|
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["target_avg_price"]
|
|
import_avm_evaluation_item_deal.loc[:, "deal_total_price"] = import_avm_evaluation_item_deal["deal_total_price"].apply(lambda x : int(x))
|
|
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["buildarea"] * import_avm_evaluation_item_deal["calc_avg_price"]
|
|
import_avm_evaluation_item_deal.loc[:, "total_price"] = import_avm_evaluation_item_deal["total_price"].apply(lambda x : int(x))
|
|
import_community_base_price = community_base_price[community_base_price.cityname.isin(important_cities)]
|
|
normal_community_base_price = community_base_price[~(community_base_price.cityname.isin(important_cities))]
|
|
import_avm_evaluation = avm_evaluation[avm_evaluation.cityname.isin(important_cities)]
|
|
avm_evaluation = avm_evaluation.merge(city_order, on="cityname", how="left")
|
|
import_community_base_price = import_community_base_price.merge(city_order, on="cityname", how="left")
|
|
normal_community_base_price = normal_community_base_price.merge(city_order, on="cityname", how="left")
|
|
import_avm_evaluation_item_deal = import_avm_evaluation_item_deal.merge(city_order, on="cityname", how="left")
|
|
import_avm_evaluation_item_third = import_avm_evaluation_item_third.merge(city_order, on="cityname", how="left")
|
|
avm_evaluation.sort_values(by="cityrank", inplace=True)
|
|
import_community_base_price.sort_values(by="cityrank", inplace=True)
|
|
normal_community_base_price.sort_values(by="cityrank", inplace=True)
|
|
import_avm_evaluation_item_deal.sort_values(by="cityrank", inplace=True)
|
|
import_avm_evaluation_item_third.sort_values(by="cityrank", inplace=True)
|
|
|
|
file_name = f"{year_month}估值模型测试.xlsx"
|
|
aim_file_name = path + '/' + file_name
|
|
with pd.ExcelWriter(aim_file_name) as writer:
|
|
avm_evaluation.to_excel(writer, sheet_name="92个城市估值概况", columns=["cityname","community_num","price_coverage","sample_calc_coverage","ann_calc_coverage","artificial_calc_coverage","deal_test_case_num","deal_ppe15_coverage","deal_test_comm_num","deal_test_comm_coverage","deal_rate_median","deal_coefficient","third_test_comm_num","third_test_comm_coverage","third_ppe15_coverage","third_rate_median","third_coefficient"], header=["城市名称","小区量","可输出均价小区占比","样本计算占比","神经网络计算占比","人工指导价占比","成交测试案例总量","成交测试PPE15占比","成交测试小区量","成交测试小区量占比","成交比率中位数","成交比率离散系数","第三方均价测试小区量","第三方均价测试小区量占比","第三方均价测试PPE15占比","第三方比率中位数","第三方比率离散系数"], index=False)
|
|
import_community_base_price.to_excel(writer, sheet_name="测试10个城市估值数据", columns=["cityname","distname","commname","comm_avg_price","confidence","calc_type"], header=["城市","区域","小区名","估值小区均价","置信度","估值方式"],index=False)
|
|
import_avm_evaluation_item_deal.to_excel(writer, sheet_name="10个城市成交测试案例", columns=['cityname','distname', 'commname', 'unit_price', 'confidence', 'calc_type','total_price', 'deal_total_price', 'target_avg_price','pe',"dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], header=["城市","区域","小区名","修正均价","置信度","估值方式","小区估价","对标成交价","对标均价","PPE","dealdate","fitment","houseaspect","floornum","floorcnt","buildarea"], index=False)
|
|
import_avm_evaluation_item_third.to_excel(writer, sheet_name="10个城市第三方均价测试", columns=["cityname","distname","commname","avg_price","confidence","calc_type","target_avg_price","pe"], header=["城市","区域","小区名","小区均价","置信度","估值方式","对标第三方均价","PPE"],index=False)
|
|
log.info("估值结果测试结束")
|
|
def up_to_nextcloud():
|
|
"""
|
|
上传统计信息至nextcloud
|
|
"""
|
|
from nextcloud import NextCloud
|
|
NEXTCLOUD_URL = "https://nextcloud.fangjia.com"
|
|
NEXTCLOUD_USERNAME = "shenjian@fangjia.com"
|
|
NEXTCLOUD_PASSWORD = "goojia123456"
|
|
nextcloud_file_name = '共享/建行/' + file_name
|
|
nxc = NextCloud(endpoint=NEXTCLOUD_URL, user=NEXTCLOUD_USERNAME, password=NEXTCLOUD_PASSWORD)
|
|
nxc.upload_file(aim_file_name, nextcloud_file_name)
|
|
|
|
def send_msg():
|
|
infos = "【准确度测评】数据已生成完毕,请点击 https://nextcloud.fangjia.com/s/KwxGsosMcnYxdoa 查看详情"
|
|
data = {
|
|
"msgtype": "text",
|
|
"text": {
|
|
"content": infos
|
|
}
|
|
}
|
|
# 发送消息至群
|
|
requests.post(
|
|
url='https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736',
|
|
json=data)
|
|
up_to_nextcloud()
|
|
send_msg() |