@@ -4,10 +4,11 @@ import pandas as pd
|
||||
import requests
|
||||
from datetime import datetime
|
||||
from goojia_utils import dateutils, oracle
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
os.environ["NLS_LANG"] = "American_America.AL32UTF8"
|
||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
||||
|
||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/postgres", echo=False)
|
||||
|
||||
important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"]
|
||||
city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx")
|
||||
@@ -53,10 +54,17 @@ if not os.path.exists(path):
|
||||
os.makedirs(path)
|
||||
year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m"))
|
||||
log.info(f"测试月份:{year_month}")
|
||||
avm_evaluation = oracle.scan_by_sql_to_data_frame(f"SELECT * FROM avm_evaluation_gj where year_month={year_month}")
|
||||
avm_evaluation_item = oracle.scan_by_sql_to_data_frame(f"SELECT * FROM avm_evaluation_item_gj where year_month={year_month}")
|
||||
community_base_price = oracle.scan_by_sql_to_data_frame(f"SELECT commid,cityname,distname,commname, avg_price as comm_avg_price,confidence,calc_type from community_base_price_gj where year_month={year_month}")
|
||||
communities = oracle.scan_by_sql_to_data_frame(f"select COMMID from bigdata_community_gj")
|
||||
avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'}
|
||||
avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'}
|
||||
avm_evaluation = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}", con=engine)
|
||||
avm_evaluation_item = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}", con=engine)
|
||||
avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True)
|
||||
avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True)
|
||||
community_base_price = pd.read_sql(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname,
|
||||
gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''',
|
||||
con=engine)
|
||||
communities = pd.read_sql(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h", con=engine)
|
||||
|
||||
community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])]
|
||||
avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code)
|
||||
avm_evaluation_item["houseaspect"] = avm_evaluation_item["houseaspect"].map(convert_house_aspect_code)
|
||||
|
||||
@@ -7,9 +7,10 @@ import logging
|
||||
import os
|
||||
import sys
|
||||
import requests
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7"
|
||||
|
||||
engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/postgres", echo=False)
|
||||
def simple_calc(result_df):
|
||||
"""
|
||||
新增是否有样本列
|
||||
@@ -25,8 +26,9 @@ def simple_calc(result_df):
|
||||
if len(actions) == 800:
|
||||
ins = "','".join(actions)
|
||||
ins = "'" + ins + "'"
|
||||
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
|
||||
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
|
||||
simple_temp_df = pd.read_sql(
|
||||
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
|
||||
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
|
||||
|
||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||
actions.clear()
|
||||
@@ -34,8 +36,9 @@ def simple_calc(result_df):
|
||||
if len(actions) > 0:
|
||||
ins = "','".join(actions)
|
||||
ins = "'" + ins + "'"
|
||||
simple_temp_df = oracle.scan_by_sql_to_data_frame("select distinct commid from bigdata_forsale_gj where " +
|
||||
" publishtime> TO_DATE('" + month_before + "','YYYY-MM-DD') and commid in (" + ins + ")")
|
||||
simple_temp_df = pd.read_sql(
|
||||
"select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " +
|
||||
" anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine)
|
||||
simple_df = pd.concat([simple_df, simple_temp_df])
|
||||
actions.clear()
|
||||
simple_df['has_sample'] = 1
|
||||
@@ -85,8 +88,8 @@ if type == month_last:
|
||||
month_now = month_last
|
||||
month_last = month_before
|
||||
|
||||
df = oracle.scan_by_sql_to_data_frame("SELECT id, cityname, distname, commid, commname, calc_type, confidence, avg_price, year_month FROM community_base_price_gj WHERE year_month IN('"+month_now+"', '"+month_last+"') " +
|
||||
" ORDER BY cityname, distname, commid, commname, year_month")
|
||||
df = pd.read_sql("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " +
|
||||
" ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd", con=engine)
|
||||
|
||||
six_map = dict()
|
||||
five_map = dict()
|
||||
|
||||
Reference in New Issue
Block a user