From 65c86f1e421d5402bbbd9c661940a27d845388be Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 24 May 2023 16:39:12 +0800 Subject: [PATCH] test --- .../pgsql/ExportAllPgData.groovy | 94 ++++++++++--------- 1 file changed, 52 insertions(+), 42 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy index 90736c8cc..58407d269 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -23,6 +23,8 @@ class ExportAllPgData { def linux_prefix = "/home/goojia/ccb_data/evaluation/" def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + def testCitys = ['110000','130200','140200','150200','320500','320800','330100','420900','440100','610100'] + static main(args) { def task = new ExportAllPgData() task.run(args) @@ -43,12 +45,12 @@ class ExportAllPgData { // exportCommunityGuidedPrice()//随政策// // //处理小区情况 // exportCommunityStaging()//没cityid - exportBigdataCommunity()// +// exportBigdataCommunity()// // // exportCommunityBuildType()// // -// exportThirdPartyPrice2(new_dists)// -// exportBigdataForsale(new_dists) //现在按updatetime导出 + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //现在按updatetime导出 // exportBigdataDeal(new_dists) //2022-07-01 @@ -65,10 +67,10 @@ class ExportAllPgData { // exportCommunityPdescribeGj() // exportCommCaseDetailGj() // - exportInOk() -// - SSHHelperTest jschCommand = new SSHHelperTest(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); +// exportInOk() +//// +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); } def exportEvaluationRateData() { @@ -371,8 +373,12 @@ class ExportAllPgData { def rets2 = getThirdPartyDatas(sql2) allRows.addAll(rets2) }else{ - def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' "; - allRows = getThirdPartyDatas(sql) + + if(testCitys.contains(cityCode)){ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' "; + allRows = getThirdPartyDatas(sql) + } + } if(allRows && allRows.size()>0){ index = index + 1 @@ -533,9 +539,11 @@ class ExportAllPgData { def rets1 = getHouseDatas(sql1) allRows.addAll(rets1) }else{ - def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + - "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' "; - allRows = getHouseDatas(sql) + if(testCitys.contains(cityCode)) { + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' "; + allRows = getHouseDatas(sql) + } } println city +" "+allRows.size() if(allRows.size()>0){ @@ -805,39 +813,41 @@ class ExportAllPgData { for(def city : citys) { def allRows = [] def cityCode = cityCodeMap.get(city) - def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// - pgsqlInstance.eachRow(sql, { r -> - def data = [] - data << r.gujia_bss_prc_id - data << r.urbn_id - data << r.urbn_nm - data << r.gujia_admndiv_id - data << r.gujia_admndiv_nm - data << r.gujia_bds_id - data << r.comm_nm - data << r.gujiabds_clc_yrmo_val - data << r.gujia_bds_clc_avprc - data << r.gujia_comm_basic_prc - data << r.gujia_bds_clmtd - data << r.gujiabds_fwd_samp_rto - data << r.gujia_bds_mns_grd - data << r.gujia_bds_cfdrg - data << r.gujiabdsarndcmprccrlt - data << r.gujia_bds_cfdc_dgr - data << r.data_crt_tm - data << r.data_udt_tm - data << r.gujia_src_co - allRows << data - if(allRows.size()%200000==0){ + if(testCitys.contains(cityCode)){ + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + + if(allRows.size()>0){ FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); - allRows.clear() } - }); - - if(allRows.size()>0){ - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); - FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); } } CcbFtpTestUtils.ftpUpload(null, fileName, curDate)