From 85669b6bd7daea1cc6fe7c11b1657004a4f95da1 Mon Sep 17 00:00:00 2001 From: "shenjian@fangjia.com" Date: Tue, 21 Mar 2023 11:43:09 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96pgsql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- dags/jh/price/no_price_community.py | 13 ++++--------- dags/jh/price/price_deviation.py | 12 +++--------- dags/jh/price/price_evaluation.py | 19 +++++++------------ dags/jh/price/price_exception.py | 18 ++++++++---------- dags/jh/price/third_price.py | 7 +++---- 5 files changed, 25 insertions(+), 44 deletions(-) diff --git a/dags/jh/price/no_price_community.py b/dags/jh/price/no_price_community.py index 49db8d7..679b7a3 100644 --- a/dags/jh/price/no_price_community.py +++ b/dags/jh/price/no_price_community.py @@ -3,12 +3,7 @@ import logging import pandas as pd import requests from datetime import datetime -from goojia_utils import dateutils, oracle -from sqlalchemy import create_engine - -os.environ["NLS_LANG"] = "American_America.AL32UTF8" -os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" -engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) +from goojia_utils import dateutils, pgsql log = logging.getLogger(__name__) path = "/home/airflow/data/jh/price" @@ -17,9 +12,9 @@ if not os.path.exists(path): year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_date(datetime.utcnow(), "%Y%m")) log.info(f"测试月份:{year_month}") -avm_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ", con=engine) -comm_df = pd.read_sql(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ", con=engine) -price_df = pd.read_sql(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ", con=engine) +avm_df = pgsql.scan_by_sql_to_data_frame(f"select urbn_id,urbn_nm,gujia_bds_comm_num,gujiabdsvalmnycommnum from t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month} ") +comm_df = pgsql.scan_by_sql_to_data_frame(f"select urbn_id,gujia_admndiv_nm,gujia_comm_id,comm_nm from base.t0831_bigdata_community_gj_h where gujia_bds_bds_st=0 ") +price_df = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_id as gujia_comm_id,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month} ") comm_df = comm_df.merge(avm_df, on="urbn_id", how="left") comm_df = comm_df.merge(price_df, on="gujia_comm_id", how="left") no_price_comm_df = comm_df[comm_df.gujia_comm_basic_prc.isna()][['urbn_id','urbn_nm','gujia_admndiv_nm','gujia_comm_id','comm_nm']] diff --git a/dags/jh/price/price_deviation.py b/dags/jh/price/price_deviation.py index ed00a3b..c662e57 100644 --- a/dags/jh/price/price_deviation.py +++ b/dags/jh/price/price_deviation.py @@ -2,13 +2,7 @@ import os, sys import logging import pandas as pd import requests -from datetime import datetime -from goojia_utils import dateutils, oracle -from sqlalchemy import create_engine - -os.environ["NLS_LANG"] = "American_America.AL32UTF8" -os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" -engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) +from goojia_utils import dateutils, pgsql log = logging.getLogger(__name__) path = "/home/airflow/data/jh/price" @@ -27,8 +21,8 @@ if type == month_last: month_last = month_before log.info(f"测试月份:{month_now}") -price_df = pd.read_sql(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ", con=engine) -deal_df = pd.read_sql(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ", con=engine) +price_df = pgsql.scan_by_sql_to_data_frame(f"select urbn_id,urbn_nm,gujia_admndiv_nm,gujia_bds_id as gujia_comm_id,comm_nm,gujia_bds_clmtd,gujia_comm_basic_prc from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={month_now} ") +deal_df = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_mdl_id,gujia_comm_id,gujia_bds_trdtckt_prc from base.t0831_bigdata_deal_gj_h where to_char(txn_dt,'yyyyMM')='{month_last}' ") deal_price_df = deal_df.merge(price_df, on="gujia_comm_id", how="left") deal_price_df["ppe"] = round((deal_price_df["gujia_comm_basic_prc"] - deal_price_df['gujia_bds_trdtckt_prc']) / deal_price_df['gujia_bds_trdtckt_prc'], 4) deal_price_df = deal_price_df[abs(deal_price_df['ppe']) > 0.15] diff --git a/dags/jh/price/price_evaluation.py b/dags/jh/price/price_evaluation.py index 6011509..f33cece 100644 --- a/dags/jh/price/price_evaluation.py +++ b/dags/jh/price/price_evaluation.py @@ -3,12 +3,7 @@ import logging import pandas as pd import requests from datetime import datetime -from goojia_utils import dateutils, oracle -from sqlalchemy import create_engine - -os.environ["NLS_LANG"] = "American_America.AL32UTF8" -os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" -engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) +from goojia_utils import dateutils, pgsql important_cities =["北京","上海","广州","深圳","重庆","杭州","苏州","厦门","大连","呼和浩特"] city_order = pd.read_excel("/opt/airflow/dags/repo/dags/jh/para/估值模型测试城市模板.xlsx") @@ -56,14 +51,14 @@ year_month = int(sys.argv[1]) if len(sys.argv) > 1 else int(dateutils.format_dat log.info(f"测试月份:{year_month}") avm_evaluation_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_bds_yrmo': 'year_month', 'gujia_bds_comm_num': 'community_num', 'gujiabdsvalmnycommnum': 'has_price_community_num', 'gujia_prc_cvr_rate': 'price_coverage', 'gujiabdsmnldpccommnum': 'artificial_calc_num', 'gujia_mnl_drc_prc_pct': 'artificial_calc_coverage', 'gujiabdssamp_comm_num': 'sample_calc_num', 'gujiaext_samp_clc_pct': 'sample_calc_coverage', 'gujiabdsnervetncomnum': 'ann_calc_num', 'gujianervethrunclcpct': 'ann_calc_coverage', 'gujiabds_tst_case_num': 'test_num', 'gujiabdsmdltstcommnum': 'deal_test_comm_num', 'gujiatst_comm_num_pct': 'test_comm_coverage', 'gujia_ppe15_cvr_rate': 'ppe15_coverage', 'gujia_rto_mdn': 'rate_median', 'gujia_rto_lv_san_ceft': 'coefficient', 'gujiabdsmdltstcasenum': 'deal_test_case_num', 'gujiamdltstcommnumpct': 'deal_test_comm_coverage', 'gujiamdltst_ppe15_pct': 'deal_ppe15_coverage', 'gujiamdlrtolvsan_ceft': 'deal_coefficient', 'gujia_mdl_md_bit_rto': 'deal_rate_median', 'gujiabdstrdpttcasenum': 'third_test_case_num', 'gujiabdstrdpttcommnum': 'third_test_comm_num', 'gujiatrdpttcommnumpct': 'third_test_comm_coverage', 'gujiatrdpttstppe15pct': 'third_ppe15_coverage', 'gujiatrdptrolvsanceft': 'third_coefficient', 'gujiatrdpt_md_bit_rto': 'third_rate_median', 'data_crt_tm': 'createtime', 'data_udt_tm': 'updatetime', 'gujia_src_co': 'sourcecode'} avm_evaluation_item_column_map = {'gujia_fsid': 'id', 'urbn_id': 'cityid', 'urbn_nm': 'cityname', 'gujia_rgon_id': 'distid', 'gujia_rgon_nm': 'distname', 'gujia_comm_id': 'commid', 'gujia_comm_nm': 'commname', 'gujiabds_clc_yrmo_val': 'year_month', 'gujia_bds_clc_avprc': 'calc_avg_price', 'gujia_bds_avprc': 'avg_price', 'gujia_bds_clc_tp': 'calc_type', 'gujia_bds_rlb': 'confidence', 'gujiabds_to_mkprc_mny': 'target_avg_price', 'mdl_dt': 'dealdate', 'gujiabds_valt_unitprc': 'unit_price', 'gujia_area': 'buildarea', 'gujia_dcrt_val': 'fitment', 'gujia_twrd_val': 'houseaspect', 'gujia_bds_floor_num': 'floornum', 'gujiabdstot_floor_num': 'floorcnt', 'gujia_data_tp': 'data_type', 'gujia_rto': 'rate', 'gujia_mns_rto': 'pe', 'data_crt_tm': 'createtime'} -avm_evaluation = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}", con=engine) -avm_evaluation_item = pd.read_sql(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}", con=engine) +avm_evaluation = pgsql.scan_by_sql_to_data_frame(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_gj_h where gujia_bds_yrmo={year_month}") +avm_evaluation_item = pgsql.scan_by_sql_to_data_frame(f"SELECT * FROM t_app_p9pha_plda.avm_evaluation_item_gj_h where gujiabds_clc_yrmo_val={year_month}") avm_evaluation.rename(columns=avm_evaluation_column_map, inplace=True) avm_evaluation_item.rename(columns=avm_evaluation_item_column_map, inplace=True) -community_base_price = pd.read_sql(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname, - gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''', - con=engine) -communities = pd.read_sql(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h", con=engine) +community_base_price = pgsql.scan_by_sql_to_data_frame(f'''SELECT gujia_bds_id as commid,urbn_nm as cityname,gujia_admndiv_nm as distname,comm_nm as commname, + gujia_comm_basic_prc as comm_avg_price,gujia_bds_cfdc_dgr as confidence,gujia_bds_clmtd as calc_type from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val={year_month}''' + ) +communities = pgsql.scan_by_sql_to_data_frame(f"select gujia_comm_id as COMMID from base.t0831_bigdata_community_gj_h") community_base_price = community_base_price[community_base_price.commid.isin(communities["commid"])] avm_evaluation_item["fitment"] = avm_evaluation_item["fitment"].map(convert_fitment_code) diff --git a/dags/jh/price/price_exception.py b/dags/jh/price/price_exception.py index 89f4c60..cfe0817 100644 --- a/dags/jh/price/price_exception.py +++ b/dags/jh/price/price_exception.py @@ -1,16 +1,14 @@ """ 价格异常检测脚本 """ -from goojia_utils import dateutils, oracle +from goojia_utils import dateutils, pgsql import pandas as pd import logging import os import sys import requests -from sqlalchemy import create_engine -os.environ["ORACLE_HOME"] = "/home/airflow/instantclient_21_7" -engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) + def simple_calc(result_df): """ 新增是否有样本列 @@ -26,9 +24,9 @@ def simple_calc(result_df): if len(actions) == 800: ins = "','".join(actions) ins = "'" + ins + "'" - simple_temp_df = pd.read_sql( + simple_temp_df = pgsql.scan_by_sql_to_data_frame( "select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " + - " anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine) + " anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")") simple_df = pd.concat([simple_df, simple_temp_df]) actions.clear() @@ -36,9 +34,9 @@ def simple_calc(result_df): if len(actions) > 0: ins = "','".join(actions) ins = "'" + ins + "'" - simple_temp_df = pd.read_sql( + simple_temp_df = pgsql.scan_by_sql_to_data_frame( "select distinct gujia_comm_id as commid from base.t0831_bigdata_forsale_gj_h where " + - " anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")", con=engine) + " anc_tm> TO_DATE('" + month_before + "','YYYY-MM-DD') and gujia_comm_id in (" + ins + ")") simple_df = pd.concat([simple_df, simple_temp_df]) actions.clear() simple_df['has_sample'] = 1 @@ -88,8 +86,8 @@ if type == month_last: month_now = month_last month_last = month_before -df = pd.read_sql("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " + - " ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd", con=engine) +df = pgsql.scan_by_sql_to_data_frame("SELECT gujia_bss_prc_id as id, urbn_nm as cityname, gujia_admndiv_nm as distname, gujia_bds_id as commid, comm_nm as commname, gujia_bds_clmtd as calc_type, gujia_bds_cfdc_dgr as confidence, gujia_comm_basic_prc as avg_price, gujiabds_clc_yrmo_val as year_month FROM t_app_p9pha_plda.avm_community_base_price_gj_h WHERE gujiabds_clc_yrmo_val IN('"+month_now+"', '"+month_last+"') " + + " ORDER BY urbn_nm, gujia_admndiv_nm, gujia_bds_id, comm_nm, gujia_bds_clmtd") six_map = dict() five_map = dict() diff --git a/dags/jh/price/third_price.py b/dags/jh/price/third_price.py index 5f01329..8db491d 100644 --- a/dags/jh/price/third_price.py +++ b/dags/jh/price/third_price.py @@ -1,5 +1,5 @@ from datetime import datetime -from goojia_utils import MAIN_MONGO_CLIENT, dateutils +from goojia_utils import MAIN_MONGO_CLIENT, dateutils, pgsql from sklearn.cluster import DBSCAN import numpy as np import pandas as pd @@ -7,7 +7,7 @@ from pykalman import KalmanFilter from pymongo import UpdateOne import transbigdata as tbd from sklearn.neighbors import NearestNeighbors -from sqlalchemy import create_engine + year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), months=-2), '%Y%m')) before_year_month = int(dateutils.format_date(dateutils.add_months(datetime.now(), years=-1, months=-1), '%Y%m')) @@ -87,8 +87,7 @@ monthly_data = monthly_data.sort_values(by=["city", "region", "name"]) # community_base_price = oracle.scan_by_sql_to_data_frame(f"select commid, year_month, avg_price, calc_type,confidence, cityname from community_base_price_gj where year_month > {before_year_month}") -engine = create_engine("postgresql+psycopg2://postgres:postgres@192.168.0.236:5432/warehouse", echo=False) -community_base_price = pd.read_sql(f"select gujia_bds_id as commid,gujiabds_clc_yrmo_val as year_month, gujia_comm_basic_prc as avg_price, urbn_nm as cityname,gujia_bds_clmtd as calc_type, gujia_bds_cfdrg as confidence from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val>{before_year_month} ", con=engine) +community_base_price = pgsql.scan_by_sql_to_data_frame(f"select gujia_bds_id as commid,gujiabds_clc_yrmo_val as year_month, gujia_comm_basic_prc as avg_price, urbn_nm as cityname,gujia_bds_clmtd as calc_type, gujia_bds_cfdrg as confidence from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val>{before_year_month} ") community_base_price = community_base_price.sort_values(by=['commid', 'year_month'], ascending=[True, True]) # In[15]: