diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 9469b6d02..811ec290c 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -171,6 +171,11 @@ ojdbc14 10.2.0.4.0 + + org.postgresql + postgresql + 42.2.1 + com.yahoo.platform.yui yuicompressor diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index bfce0b81b..27ed0ad49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -28,7 +28,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataCommunityBatch extends BaseBatch{ - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10) static main(args) { runBatch(new BigdataCommunityBatch(), args); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy new file mode 100644 index 000000000..428aaa792 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlBaseEvaluation.groovy @@ -0,0 +1,623 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +class ExportPgsqlBaseEvaluation { + + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "base" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportPgsqlBaseEvaluation() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonthStr = "2022-09"//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + def new_dists = [] + +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() + + //第三方网站权重表 +// exportWebsiteRate(); + +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策// +// +//// //处理小区情况 +// exportBigdataCommunity()// +// exportCommunityStaging()//没cityid +// exportCommunityBuildType()// +// +// exportThirdPartyPrice2(lastMonth, new_dists)// + exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19 +// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01 + + + } + + def exportEvaluationRateData() { + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_w"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_ceft + data << r.gujia_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + + + def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false) + } + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,lgt,ltt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_fsid + data << r.poi_nm + data << r.gujia_tp + data << r.gujia_tp_ecd + data << r.lgt + data << r.ltt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_data_tm from "+database+".t0831_bigdata_district_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_lgc_admndiv_dsc + data << r.gujia_lgc_admndiv_cd + data << r.gujia_adiv_cd_val + data << r.gujia_data_tm + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false) + } + + def exportWebsiteRate(){ + def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_wgt,gujia_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_w"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujiadatawebst_src_id + data << r.gujia_data_webst_src + data << r.gujiadatawebstmpng_id + data << r.gujia_wgt + data << r.gujia_tp + data << r.gujia_src_co + data << r.urbn_id + data << r.gujia_urbn + data << r.data_crt_tm + data << r.data_udt_tm + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false) + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_dt + data << r.gujia_drc_prc + data << r.gujia_add_psn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityGuidedPrice(def date) { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_drc_prc, gujia_drc_prc_src_val, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_prc + data << r.gujia_drc_prc_src_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,lgt,ltt,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_bld_yr_rc,gujia_hs_use,gujia_totunits,gujia_data_tm,gujia_udt_mark_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_st_val from "+database+".t0831_bigdata_community_gj_w where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.lgt + data << r.ltt + data << r.gujiaocpoflandtotarea + data << r.gujia_bld_tot_area + data << r.gujia_grnrate + data << r.gujia_vol_rate + data << r.gujia_park_num + data << r.gujia_bld_yr_rc + data << r.gujia_hs_use + data << r.gujia_totunits + data << r.gujia_data_tm + data << r.gujia_udt_mark_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_st_val + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_w"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_instm_id + data << r.gujia_comm_id + data << r.gujiacomm_prim_prd_id + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityBuildType() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_data_tm from "+database+".t0831_bigdata_commbuildtype_gj_w where urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_tp_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_data_tm + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false) + } + + def exportThirdPartyPrice2(price_date, new_dists) { + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujia_avprc_dt_val='${price_date}' "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_dtsr_id + data << r.gujia_dtsr_dsc + data << r.gujia_avprc_dt_val + data << r.gujia_comm_avprc + data << r.gujia_comm_hgst_prc + data << r.gujia_comm_lwst_prc + data << r.data_udt_tm + allRows << data + }) + return allRows; + } + + + def exportBigdataDeal(def dealDate, def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')"; + allRows.addAll(dealData(sql)) + } + } + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false) + } + + + def dealData(def sql){ + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.urbn_id + data << r.mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujia_tot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_trdtckt_prc + data << r.gujia_mdl_tot_prc + data << r.gujia_dcl_tot_prc + data << r.gujia_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiagvlndfargrdnarea + data << r.gujia_gv_lu_tbl_area + data << r.gujia_gv_sun_tbl_area + data << r.gujiagvbaywindow_area + data << r.gujiagvarea_tower_flr + data << r.gujia_gv_grg_area + data << r.gujiagvundgdroom_area + data << r.gujia_gv_othr_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_mkdam + data << r.gujia_data_tm + allRows << data + }); + return allRows + } + + + def exportBigdataForsale(publishDate, new_dists){ + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd')"; + allRows = getHouseDatas(sql) + } + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + sqlInstance.eachRow(sql,{r-> + index = index+1 + println r.urbn_nm +" "+index + def data = [] + data << r.urbn_id + data << r.gujia_sell_id + data << r.gujia_listg_hs_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_unitprc + data << r.gujia_tot_prc + data << r.gujia_sell_pymtmod + data << r.gujia_wbt_floor + data << r.gujia_tot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_indr_stc_val + data << r.gujia_bld_stc + data << r.gujiaconstructionarea + data << r.gujia_set_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_room + data << r.gujia_dept + data << r.gujia_ktch_num + data << r.gujia_toliet + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujiawthrext_lift_ind + data << r.gujiajiladderjiactval + data << r.anc_tm + data << r.exdat + data << r.gujia_wthr_strt_ind + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.gujia_anc_psn + data << r.gujia_anc_psn_tel + data << r.gujia_dsc_inf_cntnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_apd_inf + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" +// CcbFtpUtils.ftpUpload(null, fileName, curDate) +// println tableName+" 上传Ftp完成......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy new file mode 100644 index 000000000..1508d0b9f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlEvaluationResult.groovy @@ -0,0 +1,382 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +class ExportPgsqlEvaluationResult { + + def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "t_app_p9pha_plda." + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportPgsqlEvaluationResult() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + + //计算结果相关 +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) +// +// exportCityPrice(curMonth) +// exportRegionPrice(curMonth) +// +// exportCommunityBasePrice(curMonth) +// +// exportCommunityPdescribeGj(curMonth) +// exportCommCaseDetailGj(curMonth) +// + def curDate = curMonth.toString()+"28" +// 生成MD5文件 + exportMd5() + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// ok文件 + exportOkFile() + } + + def exportAvmEvaluationGJNB(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_yrmo,gujia_comm_num,gujia_val_comm_num,gujia_prc_cvr_rate,gujiamnldrcprccommnum,gujia_mnl_drc_prc_pct,gujia_samp_comm_num,gujiaext_samp_clc_pct,gujianervethruncomnum,gujianervethrunclcpct,gujia_tst_case_num,gujia_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiamdl_tst_case_num,gujiamdl_tst_comm_num,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiatrdpttstcase_num,gujiatrdpttstcomm_num,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where gujia_yrmo = ${yearMonth} and urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_yrmo + data << r.gujia_comm_num + data << r.gujia_val_comm_num + data << r.gujia_prc_cvr_rate + data << r.gujiamnldrcprccommnum + data << r.gujia_mnl_drc_prc_pct + data << r.gujia_samp_comm_num + data << r.gujiaext_samp_clc_pct + data << r.gujianervethruncomnum + data << r.gujianervethrunclcpct + data << r.gujia_tst_case_num + data << r.gujia_tst_comm_num + data << r.gujiatst_comm_num_pct + data << r.gujia_ppe15_cvr_rate + data << r.gujia_rto_mdn + data << r.gujia_rto_lv_san_ceft + data << r.gujiamdl_tst_case_num + data << r.gujiamdl_tst_comm_num + data << r.gujiamdltstcommnumpct + data << r.gujiamdltst_ppe15_pct + data << r.gujiamdlrtolvsan_ceft + data << r.gujia_mdl_md_bit_rto + data << r.gujiatrdpttstcase_num + data << r.gujiatrdpttstcomm_num + data << r.gujiatrdpttcommnumpct + data << r.gujiatrdpttstppe15pct + data << r.gujiatrdptrolvsanceft + data << r.gujiatrdpt_md_bit_rto + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false) + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_avprc,gujia_clc_tp_val,gujia_rlb,gujia_to_mkprc_mny, to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujia_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_wbt_floor,gujia_tot_floor,gujia_data_tp,gujia_rto,gujia_mns_rto, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm from "+database+"avm_evaluation_item_gj_h where gujia_clc_yrmo = ${yearMonth} AND urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_rgon_id + data << r.gujia_rgon_nm + data << r.gujia_comm_id + data << r.gujia_comm_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_avprc + data << r.gujia_clc_tp_val + data << r.gujia_rlb + data << r.gujia_to_mkprc_mny + data << r.mdl_dt + data << r.gujia_valt_unitprc + data << r.gujia_area + data << r.gujia_dcrt_val + data << r.gujia_twrd_val + data << r.gujia_wbt_floor + data << r.gujia_tot_floor + data << r.gujia_data_tp + data << r.gujia_rto + data << r.gujia_mns_rto + data << r.data_crt_tm + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false) + } + + def exportCityPrice(month){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportRegionPrice(month){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gxda_admndiv_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + + createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportCommunityBasePrice(yearMonth){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_comm_basic_prc,gujia_clmtd_val,gujia_fwd_samp_rto,gujia_mns_grd,gujia_cfdrg,gujiaarndcommprc_crlt,gujia_cfdc_dgr_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where gujia_clc_yrmo=${yearMonth} AND urbn_id='${cityCode}'"; + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_clc_yrmo + data << r.gujia_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_clmtd_val + data << r.gujia_fwd_samp_rto + data << r.gujia_mns_grd + data << r.gujia_cfdrg + data << r.gujiaarndcommprc_crlt + data << r.gujia_cfdc_dgr_val + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false) + createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false) + } + + def exportCommunityPdescribeGj(month){ + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bss_prc,gujia_cfdc_dgr_grd,gujiart3mlistgcasenum,gujialistgcaselwstprc,gujialistgcase_md_val,gujia_listg_case_avg,gujialistgcasemax_prc,gujiaavprc_crpnd_s_mo,gujiaavgwithminvalavg,gujiaavgwithmaxvalavg,gujia_rel_err,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujia_lmth_bss_prc,gujia_wthr_mnl_ind,gujia_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujianervethruntw_prc,gujia_prc_src,gujiaabtjanlcmdbitval,gujiaabtjanligcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujia_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujia_idxn_mo_moonmo,gujiaabt1yiminmmoonmo,gujia_idxn_yr_moonmo,gujiaabt1yr_samp_scvl,gujiaarnd1kscommavprc from "+database+"avm_community_p_describe_gj_h where gujiaavprc_crpnd_s_mo=${month} AND urbn_id='${cityCode}'"; + + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_comm_id + data << r.urbn_id + data << r.urbn_nm + data << r.admndiv_ecd + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_als_dsc + data << r.gujia_bss_prc + data << r.gujia_cfdc_dgr_grd + data << r.gujiart3mlistgcasenum + data << r.gujialistgcaselwstprc + data << r.gujialistgcase_md_val + data << r.gujia_listg_case_avg + data << r.gujialistgcasemax_prc + data << r.gujiaavprc_crpnd_s_mo + data << r.gujiaavgwithminvalavg + data << r.gujiaavgwithmaxvalavg + data << r.gujia_rel_err + data << r.gujia_dvtdgr_dsc + data << r.gujia_als_prc_rng + data << r.gujiaprcminandmax_rng + data << r.gujia_dsc + data << r.gujia_lmth_bss_prc + data << r.gujia_wthr_mnl_ind + data << r.gujia_lmth_fnl_prc + data << r.data_crt_tm + data << r.gujianervethruntw_prc + data << r.gujia_prc_src + data << r.gujiaabtjanlcmdbitval + data << r.gujiaabtjanligcaseavg + data << r.gujia_indx_lbl + data << r.gujia_prim_pty_tp + data << r.gujia_comm_tp_val + data << r.gujia_ltst_mdl_prc + data << r.gujia_ltst_mdl_dt + data << r.gujia_idxn_mo_moonmo + data << r.gujiaabt1yiminmmoonmo + data << r.gujia_idxn_yr_moonmo + data << r.gujiaabt1yr_samp_scvl + data << r.gujiaarnd1kscommavprc + allRows << data + }); + } + createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false) + } + + def exportCommCaseDetailGj(month){ + def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and crterm=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_case_id + data << r.crterm + data << r.data_crt_tm + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + CcbFtpUtils.ftpUpload(null, fileName, curDate) + } + + def exportMd5(){ + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = linux_prefix + "md5.txt" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "md5.txt" + } + createFileToFtp("md5.txt",fileRows, fileName, curDate, false) + } + + + def exportOkFile(){ + def dataCodeFileName = linux_prefix + "${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = win_prefix + "${curDate}.ok" + } + createFileToFtp("ok txt", [], dataCodeFileName, curDate, false) + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) +// println tableName+" 上传Ftp完成......" + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy new file mode 100644 index 000000000..8d3d01341 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportPgsqlRonghe.groovy @@ -0,0 +1,14 @@ +package com.fangjia.script.ccbevaluation.pgsql + +class ExportPgsqlRonghe { + + static main(args) { + def task = new ExportPgsqlRonghe() + task.run(args) + } + + def run(args) { + + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy new file mode 100644 index 000000000..2e4cfff4a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/DongGuangApiTest.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.gj.test + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class DongGuangApiTest { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + def task = new DongGuangApiTest() + task.run() + } + + def run(){ + def rets = findEsData(); + println "ES 加载完成" + def datas = evaluation(rets) + println "重估完成" +// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据") + FileUtils.batchCreateFile(datas, "/home/goojia/东莞重估.txt", [], "utf-8", true); + println "txt 生成" + exportCsv("/home/goojia/东莞重估.csv", datas) + println "csv 生成" + } + + def evaluation(rets){ + def datas = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i + def paramMap = map.parameter_map + rets << paramMap + if(rets.size()%10000==0){ + println "."+ new Date() + } + }) + return rets; + } + + + def exportCsv(path, list){ + File file = new File(path); + if (file.exists()){ + file.delete(); + } + file.append("用户输入城市,") + file.append("用户输入城市区域,") + file.append("用户输入城市地址,") + file.append("用户输入城市面积,") + file.append(System.getProperty("line.separator")); + list.each{ + def d = it; + file.append(d[0] + ",","gbk") + file.append(d[1] + ",","gbk") + file.append(d[2] + ",","gbk") + file.append(d[3] + ",","gbk") + if(d[4]){ + file.append(d[4]+",","gbk") + }else{ + file.append(",") + + } + + if(d[5]){ + file.append(d[5] +",","gbk") + }else{ + file.append(",") + + } + + + if(d[6]){ + file.append(d[6] +",","gbk") + }else{ + file.append(",") + + } + if(d[7]){ + file.append(d[7]+",","gbk") + }else{ + file.append(",") + } + file.append(System.getProperty("line.separator")); + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index c30a1353a..242e1b8a1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -221,6 +221,9 @@ class ElasticUtils { searchHit.getSourceAsMap().put("id", searchHit.getId()); results << searchHit.getSourceAsMap() } + if(results.size()%100000==0){ + println "."+ new Date() + } SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy index 7c37cae97..0d0647082 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/OpenApiClient.groovy @@ -1105,6 +1105,30 @@ public class OpenApiClient { return null } + public static JSONObject addressEvaluation(city, region, address, size){ + try { + HttpRequest request = new HttpRequest(); + Map params = new HashMap(); + params.put("city", city); + params.put("district", region); + params.put("address", address); + params.put("size", String.valueOf(size)); + params.put("token", TOKEN); + String response = request.get("http://open.fangjia.com/property/addressEvaluate", params, "UTF-8"); + JSONObject result = JSONObject.fromObject(response); + if (result.getInt("code") == 200) { + if (result.containsKey("result")) { + return result.getJSONObject("result") + } + } else if (result.getInt("code") == 500) { + println("地址估价失败:${city} ${address} ${size} ${result}") + } + } catch (Exception e) { + e.printStackTrace(); + } + return null + } + public static JSONObject evaluateDistrictPriceByAddress(String city, String address, double size) { try { HttpRequest request = new HttpRequest();