This commit is contained in:
2023-01-05 00:14:06 +08:00
parent e9ac78eff3
commit 03feeb1f0a
8 changed files with 1207 additions and 1 deletions
+5
View File
@@ -171,6 +171,11 @@
<artifactId>ojdbc14</artifactId>
<version>10.2.0.4.0</version>
</dependency>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.1</version>
</dependency>
<dependency>
<groupId>com.yahoo.platform.yui</groupId>
<artifactId>yuicompressor</artifactId>
@@ -28,7 +28,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataCommunityBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(10)
static main(args) {
runBatch(new BigdataCommunityBatch(), args);
@@ -0,0 +1,623 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
class ExportPgsqlBaseEvaluation {
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres",
"postgres", "postgres", "org.postgresql.Driver")
def citys = null
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def database = "base"
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
static main(args) {
def task = new ExportPgsqlBaseEvaluation()
task.run(args)
}
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def lastMonthStr = "2022-09"//DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM")
def new_dists = []
// exportEvaluationRateData()
// exportBigdataPoiData()
// exportBigdataDistrictData()
//第三方网站权重表
// exportWebsiteRate();
// exportGuidedPrice()//随人工改变//
// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策//
//
//// //处理小区情况
// exportBigdataCommunity()//
// exportCommunityStaging()//没cityid
// exportCommunityBuildType()//
//
// exportThirdPartyPrice2(lastMonth, new_dists)//
exportBigdataForsale(lastMonthStr +"-19", new_dists) //2022-07-19
// exportBigdataDeal(lastMonthStr + "-01", new_dists) //2022-07-01
}
def exportEvaluationRateData() {
def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_ceft,gujia_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_w";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_ceft_id
data << r.gujia_ceft_tp
data << r.gujia_ceft
data << r.gujia_seq
data << r.gujia_urbn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false)
}
def exportBigdataPoiData() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,lgt,ltt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_fsid
data << r.poi_nm
data << r.gujia_tp
data << r.gujia_tp_ecd
data << r.lgt
data << r.ltt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false)
}
def exportBigdataDistrictData() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_data_tm from "+database+".t0831_bigdata_district_gj_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_admndiv_nm_cpa
data << r.urbn_id
data << r.gujia_natstd_cd
data << r.gujia_dspl_seq
data << r.gujia_admndiv_cgy
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_lgc_admndiv_dsc
data << r.gujia_lgc_admndiv_cd
data << r.gujia_adiv_cd_val
data << r.gujia_data_tm
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false)
}
def exportWebsiteRate(){
def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_wgt,gujia_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_w";
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujiadatawebst_src_id
data << r.gujia_data_webst_src
data << r.gujiadatawebstmpng_id
data << r.gujia_wgt
data << r.gujia_tp
data << r.gujia_src_co
data << r.urbn_id
data << r.gujia_urbn
data << r.data_crt_tm
data << r.data_udt_tm
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false)
}
def exportGuidedPrice() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_bds_id
data << r.comm_nm
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_drc_dt
data << r.gujia_drc_prc
data << r.gujia_add_psn
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false)
}
def exportCommunityGuidedPrice(def date) {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_drc_prc, gujia_drc_prc_src_val, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}' and data_crt_tm>=to_date('${date}','yyyyMMdd')";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_bds_id
data << r.comm_nm
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_drc_prc
data << r.gujia_drc_prc_src_val
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false)
}
def exportBigdataCommunity(){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,lgt,ltt,gujiaocpoflandtotarea,gujia_bld_tot_area,gujia_grnrate,gujia_vol_rate,gujia_park_num,gujia_bld_yr_rc,gujia_hs_use,gujia_totunits,gujia_data_tm,gujia_udt_mark_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_st_val from "+database+".t0831_bigdata_community_gj_w where urbn_id='${cityCode}' ";//where cityId in (350200,150100)
sqlInstance.eachRow(sql,{r->
def data = []
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_comm_id
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_dtl_adr
data << r.lgt
data << r.ltt
data << r.gujiaocpoflandtotarea
data << r.gujia_bld_tot_area
data << r.gujia_grnrate
data << r.gujia_vol_rate
data << r.gujia_park_num
data << r.gujia_bld_yr_rc
data << r.gujia_hs_use
data << r.gujia_totunits
data << r.gujia_data_tm
data << r.gujia_udt_mark_val
data << r.data_udt_tm
data << r.gujia_comm_tp_val
data << r.gujia_comm_fmnm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujiawthr_new_pan_ind
data << r.gujia_comm_intd
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.gujia_src_co
data << r.gujia_vlgtwn_str_mng
data << r.gujia_lgc_admndiv_cd
data << r.gujia_lgc_adiv
data << r.gujia_bds_st_val
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false)
}
def exportCommunityStaging() {
def allRows = []
def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_w";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_instm_id
data << r.gujia_comm_id
data << r.gujiacomm_prim_prd_id
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false)
}
def exportCommunityBuildType() {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_data_tm from "+database+".t0831_bigdata_commbuildtype_gj_w where urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_bld_tp_val
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_data_tm
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false)
}
def exportThirdPartyPrice2(price_date, new_dists) {
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} ";
def rets1 = getThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} ";
def rets2 = getThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujia_avprc_dt_val,gujia_comm_avprc,gujia_comm_hgst_prc,gujia_comm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujia_avprc_dt_val='${price_date}' ";
allRows = getThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
index = index + 1
if(index==1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
}
def getThirdPartyDatas(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujia_dtsr_id
data << r.gujia_dtsr_dsc
data << r.gujia_avprc_dt_val
data << r.gujia_comm_avprc
data << r.gujia_comm_hgst_prc
data << r.gujia_comm_lwst_prc
data << r.data_udt_tm
allRows << data
})
return allRows;
}
def exportBigdataDeal(def dealDate, def new_dists) {
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_wbt_floor,gujia_lyr_tp_val,gujia_tot_floor_num,gujia_bld_tp_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_trdtckt_prc,gujia_mdl_tot_prc,gujia_dcl_tot_prc,gujia_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_mdl_cyc,gujiawthrext_lift_ind,gujiagvlndfargrdnarea,gujia_gv_lu_tbl_area,gujia_gv_sun_tbl_area,gujiagvbaywindow_area,gujiagvarea_tower_flr,gujia_gv_grg_area,gujiagvundgdroom_area,gujia_gv_othr_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_mkdam,gujia_data_tm " +
"from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealDate}','yyyy-MM-dd')";
allRows.addAll(dealData(sql))
}
}
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false)
}
def dealData(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_id
data << r.mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujia_tot_floor_num
data << r.gujia_bld_tp_val
data << r.gujia_bld_stc
data << r.gujiaconstructionarea
data << r.gujia_set_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_room
data << r.gujia_dept
data << r.gujia_ktch_num
data << r.gujia_toliet
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_trdtckt_prc
data << r.gujia_mdl_tot_prc
data << r.gujia_dcl_tot_prc
data << r.gujia_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiagvlndfargrdnarea
data << r.gujia_gv_lu_tbl_area
data << r.gujia_gv_sun_tbl_area
data << r.gujiagvbaywindow_area
data << r.gujiagvarea_tower_flr
data << r.gujia_gv_grg_area
data << r.gujiagvundgdroom_area
data << r.gujia_gv_othr_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_mkdam
data << r.gujia_data_tm
allRows << data
});
return allRows
}
def exportBigdataForsale(publishDate, new_dists){
def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat"
}
def index = 0
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}";
def rets1 = getHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and anc_tm>=to_date('${publishDate}','yyyy-MM-dd')";
allRows = getHouseDatas(sql)
}
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
}
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_unitprc,gujia_tot_prc,gujia_sell_pymtmod,gujia_wbt_floor,gujia_tot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,gujiaconstructionarea,gujia_set_innr_area,gujia_acc_tp_val,gujia_room,gujia_dept,gujia_ktch_num,gujia_toliet,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt_ind,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
def rets2 = getHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true);
}
}
println "BIGDATA_FORSALE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
println "BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def allRows = []
def index = 0
sqlInstance.eachRow(sql,{r->
index = index+1
println r.urbn_nm +" "+index
def data = []
data << r.urbn_id
data << r.gujia_sell_id
data << r.gujia_listg_hs_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_unitprc
data << r.gujia_tot_prc
data << r.gujia_sell_pymtmod
data << r.gujia_wbt_floor
data << r.gujia_tot_floor_num
data << r.gujia_bld_tp_val
data << r.gujia_indr_stc_val
data << r.gujia_bld_stc
data << r.gujiaconstructionarea
data << r.gujia_set_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_room
data << r.gujia_dept
data << r.gujia_ktch_num
data << r.gujia_toliet
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujiawthrext_lift_ind
data << r.gujiajiladderjiactval
data << r.anc_tm
data << r.exdat
data << r.gujia_wthr_strt_ind
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.gujia_anc_psn
data << r.gujia_anc_psn_tel
data << r.gujia_dsc_inf_cntnt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
data << r.gujia_apd_inf
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
});
return allRows
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
// CcbFtpUtils.ftpUpload(null, fileName, curDate)
// println tableName+" 上传Ftp完成......"
}
}
@@ -0,0 +1,382 @@
package com.fangjia.script.ccbevaluation.pgsql
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpUtils
import com.fangjia.script.ccbevaluation.ronghe.SSHHelper
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.jcraft.jsch.ChannelSftp
import groovy.sql.Sql
import org.apache.commons.codec.digest.DigestUtils
class ExportPgsqlEvaluationResult {
def sqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres",
"postgres", "postgres", "org.postgresql.Driver")
def citys = null
def cityCodeMap = CcbCommonUtils.findCityCodeValue()
def database = "t_app_p9pha_plda."
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28";
def linux_prefix = "/home/goojia/ccb_data/evaluation/"
def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\"
static main(args) {
def task = new ExportPgsqlEvaluationResult()
task.run(args)
}
def run(args) {
citys = CcbCommonUtils.getRunningCitys(args)
def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM"));
//计算结果相关
// exportAvmEvaluationGJNB(curMonth)
// exportAVMEvaluationItemGJNB(curMonth)
//
// exportCityPrice(curMonth)
// exportRegionPrice(curMonth)
//
// exportCommunityBasePrice(curMonth)
//
// exportCommunityPdescribeGj(curMonth)
// exportCommCaseDetailGj(curMonth)
//
def curDate = curMonth.toString()+"28"
// 生成MD5文件
exportMd5()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}"));
// ok文件
exportOkFile()
}
def exportAvmEvaluationGJNB(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_yrmo,gujia_comm_num,gujia_val_comm_num,gujia_prc_cvr_rate,gujiamnldrcprccommnum,gujia_mnl_drc_prc_pct,gujia_samp_comm_num,gujiaext_samp_clc_pct,gujianervethruncomnum,gujianervethrunclcpct,gujia_tst_case_num,gujia_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiamdl_tst_case_num,gujiamdl_tst_comm_num,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiatrdpttstcase_num,gujiatrdpttstcomm_num,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where gujia_yrmo = ${yearMonth} and urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_yrmo
data << r.gujia_comm_num
data << r.gujia_val_comm_num
data << r.gujia_prc_cvr_rate
data << r.gujiamnldrcprccommnum
data << r.gujia_mnl_drc_prc_pct
data << r.gujia_samp_comm_num
data << r.gujiaext_samp_clc_pct
data << r.gujianervethruncomnum
data << r.gujianervethrunclcpct
data << r.gujia_tst_case_num
data << r.gujia_tst_comm_num
data << r.gujiatst_comm_num_pct
data << r.gujia_ppe15_cvr_rate
data << r.gujia_rto_mdn
data << r.gujia_rto_lv_san_ceft
data << r.gujiamdl_tst_case_num
data << r.gujiamdl_tst_comm_num
data << r.gujiamdltstcommnumpct
data << r.gujiamdltst_ppe15_pct
data << r.gujiamdlrtolvsan_ceft
data << r.gujia_mdl_md_bit_rto
data << r.gujiatrdpttstcase_num
data << r.gujiatrdpttstcomm_num
data << r.gujiatrdpttcommnumpct
data << r.gujiatrdpttstppe15pct
data << r.gujiatrdptrolvsanceft
data << r.gujiatrdpt_md_bit_rto
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28";
def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat"
fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false)
createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false)
}
def exportAVMEvaluationItemGJNB(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_avprc,gujia_clc_tp_val,gujia_rlb,gujia_to_mkprc_mny, to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujia_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_wbt_floor,gujia_tot_floor,gujia_data_tp,gujia_rto,gujia_mns_rto, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm from "+database+"avm_evaluation_item_gj_h where gujia_clc_yrmo = ${yearMonth} AND urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_rgon_id
data << r.gujia_rgon_nm
data << r.gujia_comm_id
data << r.gujia_comm_nm
data << r.gujia_clc_yrmo
data << r.gujia_clc_avprc
data << r.gujia_avprc
data << r.gujia_clc_tp_val
data << r.gujia_rlb
data << r.gujia_to_mkprc_mny
data << r.mdl_dt
data << r.gujia_valt_unitprc
data << r.gujia_area
data << r.gujia_dcrt_val
data << r.gujia_twrd_val
data << r.gujia_wbt_floor
data << r.gujia_tot_floor
data << r.gujia_data_tp
data << r.gujia_rto
data << r.gujia_mns_rto
data << r.data_crt_tm
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat"
fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false)
createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false)
}
def exportCityPrice(month){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
data << r.gujia_clc_avprc
data << r.gujia_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false)
}
def exportRegionPrice(month){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where gujia_clc_yrmo=${month} AND urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gxda_admndiv_nm
data << r.gujia_clc_yrmo
data << r.gujia_clc_avprc
data << r.gujia_urbn_avprc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat"
}
createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false)
}
def exportCommunityBasePrice(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujia_clc_yrmo,gujia_clc_avprc,gujia_comm_basic_prc,gujia_clmtd_val,gujia_fwd_samp_rto,gujia_mns_grd,gujia_cfdrg,gujiaarndcommprc_crlt,gujia_cfdc_dgr_val,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where gujia_clc_yrmo=${yearMonth} AND urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujia_clc_yrmo
data << r.gujia_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_clmtd_val
data << r.gujia_fwd_samp_rto
data << r.gujia_mns_grd
data << r.gujia_cfdrg
data << r.gujiaarndcommprc_crlt
data << r.gujia_cfdc_dgr_val
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat"
}
createFileToFtp("COMMUNITY_BASE_PRICE_GJNB",allRows, fileName, curDate, false)
createFileToFtp("COMMUNITY_BASE_PRICE_GJ",allRows, fileName1, curDate, false)
}
def exportCommunityPdescribeGj(month){
def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bss_prc,gujia_cfdc_dgr_grd,gujiart3mlistgcasenum,gujialistgcaselwstprc,gujialistgcase_md_val,gujia_listg_case_avg,gujialistgcasemax_prc,gujiaavprc_crpnd_s_mo,gujiaavgwithminvalavg,gujiaavgwithmaxvalavg,gujia_rel_err,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujia_lmth_bss_prc,gujia_wthr_mnl_ind,gujia_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujianervethruntw_prc,gujia_prc_src,gujiaabtjanlcmdbitval,gujiaabtjanligcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujia_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujia_idxn_mo_moonmo,gujiaabt1yiminmmoonmo,gujia_idxn_yr_moonmo,gujiaabt1yr_samp_scvl,gujiaarnd1kscommavprc from "+database+"avm_community_p_describe_gj_h where gujiaavprc_crpnd_s_mo=${month} AND urbn_id='${cityCode}'";
sqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_comm_id
data << r.urbn_id
data << r.urbn_nm
data << r.admndiv_ecd
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_als_dsc
data << r.gujia_bss_prc
data << r.gujia_cfdc_dgr_grd
data << r.gujiart3mlistgcasenum
data << r.gujialistgcaselwstprc
data << r.gujialistgcase_md_val
data << r.gujia_listg_case_avg
data << r.gujialistgcasemax_prc
data << r.gujiaavprc_crpnd_s_mo
data << r.gujiaavgwithminvalavg
data << r.gujiaavgwithmaxvalavg
data << r.gujia_rel_err
data << r.gujia_dvtdgr_dsc
data << r.gujia_als_prc_rng
data << r.gujiaprcminandmax_rng
data << r.gujia_dsc
data << r.gujia_lmth_bss_prc
data << r.gujia_wthr_mnl_ind
data << r.gujia_lmth_fnl_prc
data << r.data_crt_tm
data << r.gujianervethruntw_prc
data << r.gujia_prc_src
data << r.gujiaabtjanlcmdbitval
data << r.gujiaabtjanligcaseavg
data << r.gujia_indx_lbl
data << r.gujia_prim_pty_tp
data << r.gujia_comm_tp_val
data << r.gujia_ltst_mdl_prc
data << r.gujia_ltst_mdl_dt
data << r.gujia_idxn_mo_moonmo
data << r.gujiaabt1yiminmmoonmo
data << r.gujia_idxn_yr_moonmo
data << r.gujiaabt1yr_samp_scvl
data << r.gujiaarnd1kscommavprc
allRows << data
});
}
createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false)
}
def exportCommCaseDetailGj(month){
def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat"
}
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and crterm=${month}";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.gujia_fsid
data << r.urbn_id
data << r.gujia_comm_id
data << r.gujia_case_id
data << r.crterm
data << r.data_crt_tm
allRows << data
if(allRows.size()%200000==0){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
allRows.clear()
}
});
if(allRows.size()>0){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true);
}
}
CcbFtpUtils.ftpUpload(null, fileName, curDate)
}
def exportMd5(){
Vector files = CcbFtpUtils.listFileNames(curDate)
Iterator fileVec = files.iterator()
def fileRows = []
while(fileVec.hasNext()){
ChannelSftp.LsEntry file = fileVec.next()
if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){
fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename())
}
}
def fileName = linux_prefix + "md5.txt"
if(CommonDatautils.isWindow()){
fileName = win_prefix + "md5.txt"
}
createFileToFtp("md5.txt",fileRows, fileName, curDate, false)
}
def exportOkFile(){
def dataCodeFileName = linux_prefix + "${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = win_prefix + "${curDate}.ok"
}
createFileToFtp("ok txt", [], dataCodeFileName, curDate, false)
}
def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend);
println tableName+" 数据文件输出完成......"
CcbFtpUtils.ftpUpload(null, fileName, curDate)
// println tableName+" 上传Ftp完成......"
}
}
@@ -0,0 +1,14 @@
package com.fangjia.script.ccbevaluation.pgsql
class ExportPgsqlRonghe {
static main(args) {
def task = new ExportPgsqlRonghe()
task.run(args)
}
def run(args) {
}
}
@@ -0,0 +1,155 @@
package com.fangjia.script.gj.test
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class DongGuangApiTest {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
static main(args) {
def task = new DongGuangApiTest()
task.run()
}
def run(){
def rets = findEsData();
println "ES 加载完成"
def datas = evaluation(rets)
println "重估完成"
// ExcelUtils.write("D:\\数据\\东莞银行\\东莞重估.xlsx", datas, "数据")
FileUtils.batchCreateFile(datas, "/home/goojia/东莞重估.txt", [], "utf-8", true);
println "txt 生成"
exportCsv("/home/goojia/东莞重估.csv", datas)
println "csv 生成"
}
def evaluation(rets){
def datas = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def dist = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def ret = OpenApiClient.addressEvaluation(dist.city, dist.district, dist.name, dist.size)
if(ret && ret.avgPrice){
def row = []
row << dist.city
row << dist.district
row << dist.name
row << dist.size
row << ret.city
row << ret.district
row << ret.name
row << ret.avgPrice
datas << row
}else{
def row = []
row << dist.city
row << dist.district
row << dist.name
row << dist.size
datas << row
}
println totalCount.get() +"/"+rets.size()
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return datas;
}
def findEsData(){
def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01")
QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime())
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007713'))
.must(QueryBuilders.termQuery("request_uri", '/property/evaluate'))
.must(date_range_cdate)
def rets = new CopyOnWriteArrayList()
// def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder);
// if(datas){
// for (int k=0;k<datas.size();k++) {
// def map = datas[k];
// def paramMap = map.parameter_map
// rets << paramMap
// }
// }
ElasticUtils.executeQuery("192.168.0.206", "open_uri_log", queryBuilder, { Map map ->
def paramMap = map.parameter_map
rets << paramMap
if(rets.size()%10000==0){
println "."+ new Date()
}
})
return rets;
}
def exportCsv(path, list){
File file = new File(path);
if (file.exists()){
file.delete();
}
file.append("用户输入城市,")
file.append("用户输入城市区域,")
file.append("用户输入城市地址,")
file.append("用户输入城市面积,")
file.append(System.getProperty("line.separator"));
list.each{
def d = it;
file.append(d[0] + ",","gbk")
file.append(d[1] + ",","gbk")
file.append(d[2] + ",","gbk")
file.append(d[3] + ",","gbk")
if(d[4]){
file.append(d[4]+",","gbk")
}else{
file.append(",")
}
if(d[5]){
file.append(d[5] +",","gbk")
}else{
file.append(",")
}
if(d[6]){
file.append(d[6] +",","gbk")
}else{
file.append(",")
}
if(d[7]){
file.append(d[7]+",","gbk")
}else{
file.append(",")
}
file.append(System.getProperty("line.separator"));
}
}
}
@@ -221,6 +221,9 @@ class ElasticUtils {
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
if(results.size()%100000==0){
println "."+ new Date()
}
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
@@ -1105,6 +1105,30 @@ public class OpenApiClient {
return null
}
public static JSONObject addressEvaluation(city, region, address, size){
try {
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
params.put("city", city);
params.put("district", region);
params.put("address", address);
params.put("size", String.valueOf(size));
params.put("token", TOKEN);
String response = request.get("http://open.fangjia.com/property/addressEvaluate", params, "UTF-8");
JSONObject result = JSONObject.fromObject(response);
if (result.getInt("code") == 200) {
if (result.containsKey("result")) {
return result.getJSONObject("result")
}
} else if (result.getInt("code") == 500) {
println("地址估价失败:${city} ${address} ${size} ${result}")
}
} catch (Exception e) {
e.printStackTrace();
}
return null
}
public static JSONObject evaluateDistrictPriceByAddress(String city, String address, double size) {
try {
HttpRequest request = new HttpRequest();