diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 8041a896e..953a53a6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-20") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3c0226e81..e8bd49824 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-01-01" - def date = "2023-02-01" - def endDate = "2023-03-01" - def newDate = "2023-04-01" + def startDate = "2023-02-01" + def date = "2023-03-01" + def endDate = "2023-04-01" + def newDate = "2023-05-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy index 024ad4123..640b024b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import groovy.sql.Sql @@ -11,7 +13,26 @@ class UpdateData { static main(args) { def task = new UpdateData(); - task.run() +// task.run() + task.updateBlock() + } + + def updateBlock(){ + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "block","name"]) + for(def d : dists){ + if(d.block){ + def newBlock = FiledStrand.processBlock(d.city, d.region, d.block) + if(newBlock!= d.block){ + def udata = new BasicDBObject() + udata.put("block", newBlock) + println d.city+" "+d.block +" "+newBlock + DbUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata)) + } + } + } + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy new file mode 100644 index 000000000..6e9afa65b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -0,0 +1,1020 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +class ExportAllPgData { + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + + def database = "base" + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + + static main(args) { + def task = new ExportAllPgData() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def new_dists = [] + + exportEvaluationRateData() + exportBigdataPoiData() + exportBigdataDistrictData() +// +// //第三方网站权重表 + exportWebsiteRate(); + + exportGuidedPrice()//随人工改变// + exportCommunityGuidedPrice()//随政策// + //处理小区情况 + exportCommunityStaging()//没cityid + exportBigdataCommunity()// + + exportCommunityBuildType()// + + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //现在按updatetime导出 + exportBigdataDeal(new_dists) //2022-07-01 + + + //计算结果相关 + exportAvmEvaluationGJNB() + exportAVMEvaluationItemGJNB() + + exportCityPrice() + exportRegionPrice() + + exportCommunityBasePrice() +// exportCommunityBasePrice1(curMonth) + + exportCommunityPdescribeGj() + exportCommCaseDetailGj() +// + exportInOk() +// + SSHHelperTest jschCommand = new SSHHelperTest(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + } + + def exportEvaluationRateData() { + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_bds_ceft + data << r.gujia_bds_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + + + def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false) + } + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_fsid + data << r.poi_nm + data << r.gujia_tp + data << r.gujia_tp_ecd + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_lgc_admndiv_dsc + data << r.gujia_lgc_admndiv_cd + data << r.gujia_adiv_cd_val + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false) + } + + def exportWebsiteRate(){ + def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujiadatawebst_src_id + data << r.gujia_data_webst_src + data << r.gujiadatawebstmpng_id + data << r.gujia_bds_wgt + data << r.gujia_bds_tp + data << r.gujia_src_co + data << r.urbn_id + data << r.gujia_urbn + data << r.data_crt_tm + data << r.data_udt_tm + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false) + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_dt + data << r.gujia_bds_drc_prc + data << r.gujia_add_psn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_drc_prc + data << r.gujia_bds_drc_prc_src + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from "+database+".t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + pgsqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_bds_st + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_instm_id + data << r.gujia_comm_id + data << r.gujiacomm_prim_prd_id + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityBuildType() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_tp_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false) + } + + def exportThirdPartyPrice2(new_dists) { + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' "; + allRows = getThirdPartyDatas(sql) + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_dtsr_id + data << r.gujia_dtsr_dsc + data << r.gujiabds_avprc_dt_val + data << r.gujia_bds_comm_avprc + data << r.gujiabdscomm_hgst_prc + data << r.gujiabdscomm_lwst_prc + data << r.data_udt_tm + allRows << data + }) + return allRows; + } + + + def exportBigdataDeal(def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'"; + allRows.addAll(dealData(sql)) + } + } + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false) + } + + + def dealData(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_bds_mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_bds_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_bds_trdtckt_prc + data << r.gujia_bds_mdl_tot_prc + data << r.gujia_bds_dcl_tot_prc + data << r.gujiabds_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_bds_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiabdslndfgrngvarea + data << r.gujiabdslutbl_gv_area + data << r.gujiabdssuntblgv_area + data << r.gujiabdsbayindowgarea + data << r.gujiabdstowerfrgvarea + data << r.gujia_bds_grg_gv_area + data << r.gujiabdsundgdrmgvarea + data << r.gujiabds_othr_gv_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_mkdam + data << r.gujia_bds_data_tm_val + allRows << data + }); + return allRows + } + + + def exportBigdataForsale(new_dists){ + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' "; + allRows = getHouseDatas(sql) + } + println city +" "+allRows.size() + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + pgsqlInstance.eachRow(sql,{r-> + index = index+1 + def data = [] + data << r.urbn_id + data << r.gujia_sell_id + data << r.gujia_listg_hs_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_bds_unitprc + data << r.gujia_bds_tot_prc + data << r.gujia_sell_pymtmod + data << r.gujia_bds_wbt_floor + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_indr_stc_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujiawthrext_lift_ind + data << r.gujiajiladderjiactval + data << r.anc_tm + data << r.exdat + data << r.gujia_wthr_strt + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.gujia_anc_psn + data << r.gujia_anc_psn_tel + data << r.gujia_dsc_inf_cntnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_apd_inf + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportAvmEvaluationGJNB(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_bds_yrmo,gujia_bds_comm_num,gujiabdsvalmnycommnum,gujia_prc_cvr_rate,gujiabdsmnldpccommnum,gujia_mnl_drc_prc_pct,gujiabdssamp_comm_num,gujiaext_samp_clc_pct,gujiabdsnervetncomnum,gujianervethrunclcpct,gujiabds_tst_case_num,gujiabds_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiabdsmdltstcasenum,gujiabdsmdltstcommnum,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiabdstrdpttcasenum,gujiabdstrdpttcommnum,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_bds_yrmo + data << r.gujia_bds_comm_num + data << r.gujiabdsvalmnycommnum + data << r.gujia_prc_cvr_rate + data << r.gujiabdsmnldpccommnum + data << r.gujia_mnl_drc_prc_pct + data << r.gujiabdssamp_comm_num + data << r.gujiaext_samp_clc_pct + data << r.gujiabdsnervetncomnum + data << r.gujianervethrunclcpct + data << r.gujiabds_tst_case_num + data << r.gujiabds_tst_comm_num + data << r.gujiatst_comm_num_pct + data << r.gujia_ppe15_cvr_rate + data << r.gujia_rto_mdn + data << r.gujia_rto_lv_san_ceft + data << r.gujiabdsmdltstcasenum + data << r.gujiabdsmdltstcommnum + data << r.gujiamdltstcommnumpct + data << r.gujiamdltst_ppe15_pct + data << r.gujiamdlrtolvsan_ceft + data << r.gujia_mdl_md_bit_rto + data << r.gujiabdstrdpttcasenum + data << r.gujiabdstrdpttcommnum + data << r.gujiatrdpttcommnumpct + data << r.gujiatrdpttstppe15pct + data << r.gujiatrdptrolvsanceft + data << r.gujiatrdpt_md_bit_rto + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false) + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_avprc,gujia_bds_clc_tp,gujia_bds_rlb,gujiabds_to_mkprc_mny,to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujiabds_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_bds_floor_num,gujiabdstot_floor_num,gujia_data_tp,gujia_rto,gujia_mns_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') AS data_crt_tm from "+database+"avm_evaluation_item_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_rgon_id + data << r.gujia_rgon_nm + data << r.gujia_comm_id + data << r.gujia_comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_avprc + data << r.gujia_bds_clc_tp + data << r.gujia_bds_rlb + data << r.gujiabds_to_mkprc_mny + data << r.mdl_dt + data << r.gujiabds_valt_unitprc + data << r.gujia_area + data << r.gujia_dcrt_val + data << r.gujia_twrd_val + data << r.gujia_bds_floor_num + data << r.gujiabdstot_floor_num + data << r.gujia_data_tp + data << r.gujia_rto + data << r.gujia_mns_rto + data << r.data_crt_tm + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false) + } + + def exportCityPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportRegionPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where urbn_id='${cityCode}'"; // + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gxda_admndiv_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportCommunityBasePrice(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false) + createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false) + } + + + def exportCommunityBasePrice1(yearMonth){ + + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + } + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + } + } + + def exportCommunityPdescribeGj(){ + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from "+database+"avm_community_p_describe_gj_h where urbn_id='${cityCode}'"; + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_comm_id + data << r.urbn_id + data << r.urbn_nm + data << r.admndiv_ecd + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_als_dsc + data << r.gujia_bds_bss_prc + data << r.gujiabds_cfdc_dgr_grd + data << r.gujiabdsrt3mlgcasenum + data << r.gujiabdslistgclwstprc + data << r.gujiabdslistgcaemdval + data << r.gujiabdslistgcase_avg + data << r.gujiabdslistgcemaxprc + data << r.gujia_bds_mo_val + data << r.gujiabdsavgwminvalavg + data << r.gujiabdsavgwmaxvalavg + data << r.gujia_bds_rel_err_val + data << r.gujia_dvtdgr_dsc + data << r.gujia_als_prc_rng + data << r.gujiaprcminandmax_rng + data << r.gujia_dsc + data << r.gujiabds_lmth_bss_prc + data << r.gujiabds_wthr_mnl_val + data << r.gujiabds_lmth_fnl_prc + data << r.data_crt_tm + data << r.gujiabdsnervetuntwprc + data << r.gujia_prc_src + data << r.gujiabdsabtjlcmbitval + data << r.gujiabdsabtjlgcaseavg + data << r.gujia_indx_lbl + data << r.gujia_prim_pty_tp + data << r.gujia_comm_tp_val + data << r.gujiabds_ltst_mdl_prc + data << r.gujia_ltst_mdl_dt + data << r.gujiabdsidxnmo_moonmo + data << r.gujiabdsa1yiminmmonmo + data << r.gujiabdsidxnyr_moonmo + data << r.gujiabdsat1yspdgrscor + data << r.gujiabdsarnd1kscavprc + allRows << data + }); + } + createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false) + } + + def exportCommCaseDetailGj(){ + def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } +// for(def city : citys){ +// def cityCode = cityCodeMap.get(city) +// def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}"; +// def allRows = [] +// pgsqlInstance.eachRow(sql,{r-> +// def data = [] +// data << r.gujia_fsid +// data << r.urbn_id +// data << r.gujia_comm_id +// data << r.gujia_case_id +// data << r.gujia_bds_crterm +// data << r.data_crt_tm +// allRows << data +//// if(allRows.size()%200000==0){ +//// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); +//// allRows.clear() +//// } +// }); +// +// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); +// } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def exportMd5(){ + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = linux_prefix + "md5.txt" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "md5.txt" + } + createFileToFtp("md5.txt",fileRows, fileName, curDate, false) + } + + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) + } + } + def fileName = linux_prefix + "/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "ok文件上传Ftp完成......" + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" 上传Ftp完成......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 7447a6712..cee28b12f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -128,9 +128,9 @@ class CityInfoAnaBatch extends BaseBatch { SERVER_AREA_CONN.find(cityQuery).each { allCitys.put(it.display, it); } allCitys.each {k,v -> -// if(["南京"].contains(k.toString())){ + if(["漳州"].contains(k.toString())){ CITYS << k.toString(); -// } + } } // 所有城市的人口信息 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index e6cdef564..1b7d3b371 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.gj +import com.fangjia.core.BlockData +import com.fangjia.core.RegionData import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.linxsh.util.AddressUtil @@ -337,11 +339,12 @@ class FiledStrand { return null } - static def processBlock(def block){ + static def processBlock(city, region, def block){ if(block){ if(block.contains("无") || block.toString().length()>10){ return null } +// block = BlockData.formatBlock(city, region, block); return block } return null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 2806a48f1..a04bae008 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CommonDatautils.findAllCity().reverse() + def citys = ["漳州"]//CommonDatautils.findAllCity().reverse() def analysRets = [] @@ -279,7 +279,7 @@ class DistrictFieldAddBatch extends BaseBatch{ newVal = FiledStrand.processAddress(esVal, dist.city) } if(seaweedfield =="block"){ - newVal = FiledStrand.processBlock(esVal) + newVal = FiledStrand.processBlock(dist.city, dist.region, esVal) } if(seaweedfield == "build_type"){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 319e44ef4..a59b0747a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -26,8 +26,10 @@ class SeaweedDataMatchProcess { rMap.region = obj.region rMap.alias = obj.alias rMap.name = obj.name - rMap.lat = obj.lat/10000000000.0 - rMap.lng = obj.lng/10000000000.0 + if(obj.lat){ + rMap.lat = obj.lat/10000000000.0 + rMap.lng = obj.lng/10000000000.0 + } rMap.id = obj._id.toString() rMap.address = obj.address } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 91f635c3f..4961abd56 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -35,6 +35,7 @@ public class BeiKeCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + //广州,佛山 重庆 static main(args) { def task = new BeiKeCrawlByChrome(); task.run() @@ -447,7 +448,7 @@ public class BeiKeCrawlByChrome { btn.submit(); // 点击行为; Thread.sleep(10000) }catch(Exception e){ - Thread.sleep(60000) + Thread.sleep(10000) } return driver }