房源布隆过滤器

This commit is contained in:
2023-03-07 16:27:42 +08:00
parent 90733d071b
commit 11531e5a93
2 changed files with 563 additions and 26 deletions
@@ -10,6 +10,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.google.common.hash.*
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
@@ -77,6 +78,10 @@ class BigdataForsaleBatch extends BaseBatch{
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def houseIdsfilter = findAllHouseIdsByCity()
for(def city : citys){
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
@@ -84,7 +89,7 @@ class BigdataForsaleBatch extends BaseBatch{
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230218, 20230319, houseIdsfilter);
//测试通过匹配接口配房源效果
// exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118)
}
@@ -107,9 +112,9 @@ class BigdataForsaleBatch extends BaseBatch{
}
}
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def exportSellCSV(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate)
def retMap = findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter)
println city + "(原)房源总量:"+retMap.size()
def allDatas = retMap.values()
@@ -135,12 +140,11 @@ class BigdataForsaleBatch extends BaseBatch{
saveToPgSql(allDatas)
}
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate){
def findAllCommHouse(index,city, cityCodeDataMap, distCodeMap, districts, startDate, endDate, houseIdsfilter){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
println city +" 小区量:"+districts.size()
def houseIds = findAllHouseIdsByCity(cityCodeDataMap.get(city))
println city +" ${city} 库已存在房源:"+districts.size()
def retMap = new ConcurrentHashMap()
@@ -170,7 +174,7 @@ class BigdataForsaleBatch extends BaseBatch{
def estateType = CcbCodeUtils.parseHouseEstateType(rMap.build_type);
def struCode = CcbCodeUtils.getBuildStructureCode(rMap.build_structure);
for(def m : ret){
if(!houseIds.contains(m.houseId)){
if(!houseIdsfilter.mightContain(m.houseId)){
m.city_code = cityCode
m.region = areaData.district
m.region_code = regionCode
@@ -1096,14 +1100,18 @@ class BigdataForsaleBatch extends BaseBatch{
return houseIds;
}
def findAllHouseIdsByCity(def cityCode){
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE cityid='"+cityCode+"'";
def houseIds = new HashSet()
sqlInstance.eachRow(sql,{it->
houseIds << it.FORSALEID
def findAllHouseIdsByCity(){
// 创建布隆过滤器对象
BloomFilter<String> filter = BloomFilter.create(Funnels.integerFunnel(), 200000000, 0.01);
// def houseIds = new HashSet()
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h",{it->
filter.put(it.gujia_sell_id)
});
return houseIds;
pgsqlInstance.eachRow("SELECT gujia_sell_id FROM base.t0831_bigdata_forsale_gj_h_2021",{it->
filter.put(it.gujia_sell_id)
});
return filter;
}
@@ -20,6 +20,10 @@ class ExportDistrictRongHe extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new ExportDistrictRongHe(), args);
}
@@ -29,7 +33,6 @@ class ExportDistrictRongHe extends BaseBatch{
@Override
void process(Object args) {
def queryDate = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM").toInteger()
@@ -37,8 +40,10 @@ class ExportDistrictRongHe extends BaseBatch{
def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def new_dists = []//["350604","350605","410307","410308","450181"]
//按天导(每天一个文件夹)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
// exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
exportPgBigdataDealFJW(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"), new_dists)
if(curDate.toString().endsWith("10")){
exportBigdataCommunityGJRH()//
@@ -63,26 +68,30 @@ class ExportDistrictRongHe extends BaseBatch{
//因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供
def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
exportThirdPartyPrice2(thirdPartyMonth, new_dists)
// exportThirdPartyPrice2(thirdPartyMonth, new_dists)
exportPgThirdPartyPrice2(thirdPartyMonth, new_dists)
//1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据
exportCommunityBasePrice(lastMonth)
exportCommunityPdescribeGj(lastMonth)
exportGuideStatistics(lastMonth)
exportBigdataSupplierStatistics(lastMonth)
exportBigdataModelingBase(lastMonth)
// exportCommunityBasePrice(lastMonth)
// exportCommunityPdescribeGj(lastMonth)
// exportGuideStatistics(lastMonth)
// exportBigdataSupplierStatistics(lastMonth)
// exportBigdataModelingBase(lastMonth)
exportPgCommunityBasePrice(lastMonth)
exportPgCommunityPdescribeGj(lastMonth)
exportPgGuideStatistics(lastMonth)
exportPgBigdataSupplierStatistics(lastMonth)
exportPgBigdataModelingBase(lastMonth)
//2月8号 临时导出苏州一年的数据
exportBigdataForsale("2022-07-01", new_dists) //现在按updatetime导出
exportPgBigdataForsale("2023-01-01",DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()), "yyyy-MM-dd"),new_dists)//融合按发布时间导,估值按updatetime导出
}
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
}
//
@@ -579,6 +588,46 @@ class ExportDistrictRongHe extends BaseBatch{
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def exportPgBigdataDealFJW(curDate, dealdate, new_dists){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
if(curDate.toString().endsWith("10")){
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>=to_date('${dealdate}','yyyy-MM-dd') and gujia_admndiv_id not in ${newstr}"
def rets1 = dealData(sql1)
def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}"
def rets2 = dealData(sql2)
allRows.addAll(rets1)
allRows.addAll(rets2)
}else{
def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " +
"from base.t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and txn_dt>='${dealdate}'";
allRows.addAll(dealData(sql))
}
}
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def createDeal(sql){
def allRows = []
sqlInstance.eachRow(sql,{r->
@@ -658,6 +707,87 @@ class ExportDistrictRongHe extends BaseBatch{
return allRows;
}
def dealData(def sql){
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.gujia_bds_mdl_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_str
data << r.gujia_hsno
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bds_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_prd_val
data << r.gujia_bds_wbt_floor
data << r.gujia_lyr_tp_val
data << r.gujiabdstot_floor_num
if(r.gujia_bld_tp_val){
def arrs = r.gujia_bld_tp_val.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.gujia_bld_stc
data << r.guiabconstructionarea
data << r.gujiabdsset_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_bds_room_num
data << r.gujia_bds_dept_num
data << r.gujia_bds_ktch_num
data << r.gujia_bds_hlt_btw_num
data << r.gujia_hs_use
data << r.gujia_indr_stc_val
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujia_ases_bt_ln_num
data << r.gujia_bds_trdtckt_prc
data << r.gujia_bds_mdl_tot_prc
data << r.gujia_bds_dcl_tot_prc
data << r.gujiabds_ases_tot_prc
data << r.gujiasigntonlinectrid
data << r.txn_dt
data << r.gujia_txn_tp_val
data << r.gujia_sell_pymtmod
data << r.gujia_bds_mdl_cyc
data << r.gujiawthrext_lift_ind
data << r.gujiabdslndfgrngvarea
data << r.gujiabdslutbl_gv_area
data << r.gujiabdssuntblgv_area
data << r.gujiabdsbayindowgarea
data << r.gujiabdstowerfrgvarea
data << r.gujia_bds_grg_gv_area
data << r.gujiabdsundgdrmgvarea
data << r.gujiabds_othr_gv_area
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
// data << r.gujia_bds_mkdam
data << r.gujia_bds_data_tm_val
allRows << data
});
return allRows
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = []
@@ -877,6 +1007,69 @@ class ExportDistrictRongHe extends BaseBatch{
CcbFtpUtils.ftpUpload('rh', fileName, curDate)
}
def exportPgThirdPartyPrice2(price_date, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "10";
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
if (CommonDatautils.isWindow()) {
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def index = 0
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists) {
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujiabds_avprc_dt_val=${price_date} and gujia_admndiv_id not in ${newstr} ";
def rets1 = getPgThirdPartyDatas(sql1)
allRows.addAll(rets1)
def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where " +
"urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} ";
def rets2 = getPgThirdPartyDatas(sql2)
allRows.addAll(rets2)
}else{
def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' and gujiabds_avprc_dt_val='${price_date}' ";
allRows = getPgThirdPartyDatas(sql)
}
if(allRows && allRows.size()>0){
index = index + 1
if(index==1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......"
CcbFtpUtils.ftpUpload('rh', fileName, curDate)
}
def getPgThirdPartyDatas(def sql){
def allRows = []
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_trdpt_avprc_id
data << r.urbn_id
data << r.gujia_admndiv_id
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujia_dtsr_id
data << r.gujia_dtsr_dsc
data << r.gujiabds_avprc_dt_val
data << r.gujia_bds_comm_avprc
data << r.gujiabdscomm_hgst_prc
data << r.gujiabdscomm_lwst_prc
data << r.data_udt_tm
allRows << data
})
return allRows;
}
def getThirdPartyDatas(def sql){
def allRows = []
sqlInstance.eachRow(sql, { r ->
@@ -957,6 +1150,49 @@ class ExportDistrictRongHe extends BaseBatch{
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportPgCommunityBasePrice(yearMonth){
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from t_app_p9pha_plda.avm_community_base_price_gj_h where gujiabds_clc_yrmo_val=${yearMonth} AND urbn_id='${cityCode}'";//
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_bss_prc_id
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_bds_id
data << r.comm_nm
data << r.gujiabds_clc_yrmo_val
data << r.gujia_bds_clc_avprc
data << r.gujia_comm_basic_prc
data << r.gujia_bds_clmtd
data << r.gujiabds_fwd_samp_rto
data << r.gujia_bds_mns_grd
data << r.gujia_bds_cfdrg
data << r.gujiabdsarndcmprccrlt
data << r.gujia_bds_cfdc_dgr
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
allRows << data
});
}
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat"
}
def headers = []
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......"
}
def exportCommunityPdescribeGj(month){
@@ -1031,6 +1267,74 @@ class ExportDistrictRongHe extends BaseBatch{
println "community_p_describe_gj 上传Ftp完成......"
}
def exportPgCommunityPdescribeGj(month){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from t_app_p9pha_plda.avm_community_p_describe_gj_h where gujia_bds_mo_val=${month} AND urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_fsid
data << r.gujia_comm_id
data << r.urbn_id
data << r.urbn_nm
data << r.admndiv_ecd
data << r.gujia_admndiv_nm
data << r.comm_nm
data << r.gujia_als_dsc
data << r.gujia_bds_bss_prc
data << r.gujiabds_cfdc_dgr_grd
data << r.gujiabdsrt3mlgcasenum
data << r.gujiabdslistgclwstprc
data << r.gujiabdslistgcaemdval
data << r.gujiabdslistgcase_avg
data << r.gujiabdslistgcemaxprc
data << r.gujia_bds_mo_val
data << r.gujiabdsavgwminvalavg
data << r.gujiabdsavgwmaxvalavg
data << r.gujia_bds_rel_err_val
data << r.gujia_dvtdgr_dsc
data << r.gujia_als_prc_rng
data << r.gujiaprcminandmax_rng
data << r.gujia_dsc
data << r.gujiabds_lmth_bss_prc
data << r.gujiabds_wthr_mnl_val
data << r.gujiabds_lmth_fnl_prc
data << r.data_crt_tm
data << r.gujiabdsnervetuntwprc
data << r.gujia_prc_src
data << r.gujiabdsabtjlcmbitval
data << r.gujiabdsabtjlgcaseavg
data << r.gujia_indx_lbl
data << r.gujia_prim_pty_tp
data << r.gujia_comm_tp_val
data << r.gujiabds_ltst_mdl_prc
data << r.gujia_ltst_mdl_dt
data << r.gujiabdsidxnmo_moonmo
data << r.gujiabdsa1yiminmmonmo
data << r.gujiabdsidxnyr_moonmo
data << r.gujiabdsat1yspdgrscor
data << r.gujiabdsarnd1kscavprc
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "community_p_describe_gj 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "community_p_describe_gj 上传Ftp完成......"
}
def exportGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
@@ -1072,6 +1376,75 @@ class ExportDistrictRongHe extends BaseBatch{
}
def exportPgGuideStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_GUIDE_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "SELECT * FROM t_app_p9pha_plda.avm_guide_statistics where gujia_clc_yrmo='${time}' and urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.gujia_model_type
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
data << r.gujia_comm_sum_count
data << r.gujia_comm_count
data << r.gujia_ratio
data << r.gujia_type
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "guide_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "guide_statistics 上传Ftp完成......"
}
def exportPgBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_SUPPLIER_STATISTICS_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "SELECT * FROM t_app_p9pha_plda.avm_bigdata_supplier_statistics WHERE gujia_clc_yrmo='${time}' and urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
data << r.gujia_data_sour
data << r.gujia_third_comm_count
data << r.gujia_model_type
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_supplier_statistics 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "bigdata_supplier_statistics 上传Ftp完成......"
}
def exportBigdataSupplierStatistics(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
@@ -1110,6 +1483,7 @@ class ExportDistrictRongHe extends BaseBatch{
println "bigdata_supplier_statistics 上传Ftp完成......"
}
def exportBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
@@ -1153,6 +1527,44 @@ class ExportDistrictRongHe extends BaseBatch{
println "bigdata_modeling_base 上传Ftp完成......"
}
def exportPgBigdataModelingBase(time){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_MODELING_BASE_ADD_${curDate}_0001.dat"
}
def headers = []
def allRows = []
for(def city : citys) {
def cityCode = cityCodeMap.get(city)
def sql = "select * from t_app_p9pha_plda.avm_bigdata_modeling_base_gj where gujia_clc_yrmo='${time}' and urbn_id='${cityCode}'";
pgsqlInstance.eachRow(sql, { r ->
def data = []
data << r.urbn_id
data << r.urbn_nm
data << r.gujia_clc_yrmo
data << r.gujia_esti_deal_ct
data << r.gujia_esti_comm_ct
data << r.gujia_deal_ct
data << r.gujia_deal_comm_ct
data << r.gujia_sale_ct
data << r.gujia_sale_comm_ct
data << r.gujia_model_type
data << r.gujia_clean_type
allRows << data
});
}
println allRows.size()
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
println "bigdata_modeling_base 数据文件输出完成......"
CcbFtpUtils.ftpUpload("rh", fileName, curDate)
println "bigdata_modeling_base 上传Ftp完成......"
}
def exportBigdataCityPrice(queryDate){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_CITYPRICE_FJW_ADD_${curDate}_0001.dat"
@@ -1363,6 +1775,54 @@ class ExportDistrictRongHe extends BaseBatch{
println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def exportPgBigdataForsale(publishDate,endDate, new_dists){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_FORSALE_FJW_ADD_${curDate}_0001.dat"
}
def headers = []//["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
def index = 0
for(def city : citys){
def cityCode = cityCodeMap.get(city)
def allRows = []
if(new_dists){
def newstr = "('"+new_dists.join("','")+"')"
def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and Anc_Tm>='${publishDate}' and Anc_Tm<'${endDate}' and gujia_admndiv_id not in ${newstr}";
def rets1 = getPgHouseDatas(sql1)
allRows.addAll(rets1)
}else{
def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and Anc_Tm>='${publishDate}' and Anc_Tm<'${endDate}'";
allRows = getPgHouseDatas(sql)
}
if(allRows.size()>0){
index = index + 1
if(index == 1){
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false);
}else{
FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
}
}
}
if(new_dists){//补充被修区域的数据
def newstr = "('"+new_dists.join("','")+"')"
def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " +
"from base.t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}"
def rets2 = getPgHouseDatas(sql2)
if(rets2.size()>0){
FileUtils.batchCreateFile(rets2, fileName, headers, "utf-8", true);
}
}
println "融合 BIGDATA_FORSALE_GJ 数据文件输出完成......"
// CcbFtpUtils.ftpUpload("rh",fileName, curDate)
println "融合 BIGDATA_FORSALE_GJ 上传Ftp完成......"
}
def getHouseDatas(sql){
def allRows = []
def index = 0
@@ -1432,6 +1892,75 @@ class ExportDistrictRongHe extends BaseBatch{
}
def getPgHouseDatas(def sql){
def allRows = []
def index = 0
pgsqlInstance.eachRow(sql,{r->
index = index+1
println r.urbn_nm +" "+index
def data = []
data << r.urbn_id
data << r.gujia_sell_id
data << r.gujia_listg_hs_id
data << r.urbn_nm
data << r.gujia_admndiv_id
data << r.gujia_admndiv_nm
data << r.gujia_comm_id
data << r.comm_nm
data << r.gujia_bld_id
data << r.bld_id
data << r.gujia_bld_nm
data << r.gujia_unit_id
data << r.unit_id
data << r.gujia_unit_nm
data << r.gujia_acc_id
data << r.gujia_dtl_adr
data << r.gujia_bds_bld_yr_rc
data << r.gujia_own_val
data << r.gujia_own_yrlmt_val
data << r.gujia_bds_unitprc
data << r.gujia_bds_tot_prc
data << r.gujia_sell_pymtmod
data << r.gujia_bds_wbt_floor
data << r.gujiabdstot_floor_num
data << r.gujia_bld_tp_val
data << r.gujia_indr_stc_val
data << r.gujia_bld_stc
data << r.guiabconstructionarea
data << r.gujiabdsset_innr_area
data << r.gujia_acc_tp_val
data << r.gujia_bds_room_num
data << r.gujia_bds_dept_num
data << r.gujia_bds_ktch_num
data << r.gujia_bds_hlt_btw_num
data << r.gujia_twrd_val
data << r.gujia_dcrt_sttn
data << r.gujiawthrext_lift_ind
data << r.gujiajiladderjiactval
data << r.anc_tm
data << r.exdat
data << r.gujia_wthr_strt
data << r.gujia_hs_src_dtsrc
data << r.intr_co_id
data << r.intr_co_nm
data << r.gujia_anc_psn
data << r.gujia_anc_psn_tel
data << r.gujia_dsc_inf_cntnt
data << r.rmrk_dsc
data << r.data_crt_tm
data << r.data_udt_tm
data << r.gujia_src_co
// data << r.ATTACH //融合房源没有这个字段
allRows << data
// if(allRows.size()%200000==0){
// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true);
// allRows.clear()
// }
});
return allRows
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"