Merge branch 'script_gj' into 'master'

update

See merge request !452
This commit is contained in:
2022-11-17 09:52:52 +08:00
4 changed files with 751 additions and 4 deletions
@@ -0,0 +1,747 @@
package com.fangjia.script.ccbevaluation.ronghe
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.ccbevaluation.CcbFtpTestUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.ccb.FileUtils
import com.fangjia.script.util.DateUtils
import groovy.sql.Sql
import java.sql.ResultSet
import java.sql.ResultSetMetaData
@BatchOwner(Watcher.GaoJuan)
class ExportDistrictRongHeTest extends BaseBatch{
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
runBatch(new ExportDistrictRongHeTest(), args);
}
@Override
void process(Object args) {
def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}"
// if(CommonDatautils.isWindow()){
// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}"
// }
CcbFtpTestUtils.delete(fileName)
exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd"))
exportBigdataCommunityGJRH()
exportBigdataBuildingFJW()
exportBigdataCommfeeGj()
exportBigdataCommbuildstruGJ()
exportBigdataCommhousetypeGJ()
exportBigdataCommbuildyearGj()
exportBigdataCommhouseuseGJ()
exportBigdataPositionGJ()
exportBigdataDistrictFJW()
exportBigdataAliasFJW()
exportBigdataBuildprojFjw()
exportBigdataCommlayoutpicFjw()
exportEvaluateRate()
SSHHelper jschCommand = new SSHHelper();
System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
exportOk(curDate)
}
//
// def runByCity(){
// def citys = CcbCommonUtils.citys;
// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
//
//
//// //检查ok文件是否上传
//// exportAllBigdataDealFJW()
//// for(def city : citys) {
//// def cityCode = cityCodeDataMap.get(city)
////
// //按天导(一个文件夹下)
// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
// //按天导(每天一个文件夹)
// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd"))
//
//// exportBigdataCommunityGJRH(cityCode)
//// exportBigdataBuildingFJW(cityCode)
//// exportBigdataCommfeeGj(cityCode)
////
//// exportBigdataCommbuildstruGJ(cityCode)
//// exportBigdataCommhousetypeGJ(cityCode)
//// exportBigdataCommbuildyearGj(cityCode)
//// exportBigdataCommhouseuseGJ(cityCode)
//// exportBigdataPositionGJ(cityCode)
//// exportBigdataDistrictFJW(cityCode)
//// exportBigdataAliasFJW(cityCode)
////
//// exportBigdataBuildprojFjw(cityCode)
//// exportBigdataCommlayoutpicFjw(cityCode)
//// }
//// exportEvaluateRate()
//
////
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
// SSHHelper jschCommand = new SSHHelper();
// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}"));
//
// exportOk()
// }
def exportBigdataCommunityGJRH(cityCode){
def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMMNAME
data << r.COMMALIAS
data << r.DISTID
data << r.STREET
data << r.STREETNO
data << r.ADDRESS
data << r.LANDAREA
data << r.BUILDAREA
data << r.COMMTYPE
data << r.IFNEW
data << r.GREENINGRATE
data << r.PLOTRATIO
data << r.PARKINGCNT
data << ''
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......"
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildstruGJ(){
def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDSTRU
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......"
}
def exportBigdataCommhousetypeGJ(cityCode){
def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSETYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......"
}
def exportBigdataCommbuildyearGj(){
def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.BUILDYEAR
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......"
}
def exportBigdataCommfeeGj(){
def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.COMPID
data << r.FEE
data << r.HOUSEUSE
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.FEETYPE
data << r.NOTE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMFEE_FJW 上传Ftp完成......"
}
def exportBigdataCommhouseuseGJ(){
def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.COMMID
data << r.HOUSEUSE
data << r.HOUSECNT
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......"
}
def exportBigdataPositionGJ(){
def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.OBJECTTYPE
data << r.OBJECTID
data << r.POSITIONTYPE
data << r.SEQNO
data << r.LONGITUDE
data << r.LATITUDE
data << r.ALTITUDE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_POSITION_FJW 上传Ftp完成......"
}
def exportBigdataDistrictFJW(){
def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.DISTID
data << r.DISTNAME
data << r.DISTSPELL
data << r.CITYID
data << r.DISTCODE
data << r.SEQ
data << r.DISTTYPE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DISTRICT_FJW 上传Ftp完成......"
}
def exportBigdataAliasFJW(){
def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.city_id
data << r.ca_id
data << r.address_id
data << r.address_name
data << r.status
data << r.exp_dt
data << r.version
data << r.is_key_name
data << r.data_source
data << r.description
data << r.alias_grade
data << r.addr_type_cd
data << r.ifexamine
data << r.ifdisplay
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
}
def exportBigdataDealFJW(dealdate){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def stop = false
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def index = 0
def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd")
def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd")
while(!stop){
index++
def curStartStr = curStartDateStr
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat"
}
def allRows = []
if(index==1){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
}
if(curStartDateStr == curSndDateStr){
stop = true
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh", fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}"
def curEndStr = DateUtils.add(curStartStr, 1)
curStartDateStr = curEndStr
}
}
def exportBigdataDealFJW2(curDate, dealdate){
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
def allRows = []
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}"
}
def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE<to_date('2021-12-01','yyyy-MM-dd')";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.DEALID
data << r.CITYNAME
data << r.DISTID
data << r.DISTNAME
data << r.STREET
data << r.STREETNO
data << r.COMMID
data << r.COMMNAME
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.UNITID
data << r.UNITNO
data << r.UNITNAME
data << r.HOUSEID
data << r.ADDRESS
data << r.BUILDYEAR
data << r.PROPTYPE
data << r.PROPYEARS
data << r.PERIODNUM
data << r.FLOORNUM
data << r.FLOORTYPE
data << r.FLOORCNT
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.BUILDAREA
data << r.LIVINGAREA
data << r.HOUSESTYLE
data << r.ROOMCNT
data << r.LIVINGROOMCNT
data << r.KITCHENCNT
data << r.BATHROOMCNT
data << r.HOUSEUSE
data << r.HOUSESTRU
data << r.HOUSEASPECT
data << r.FITMENT
data << r.EVALUATENO
data << r.DEALUNITPRICE
data << r.DEALTOTALPRICE
data << r.DECLAREPRICE
data << r.EVALUATEPRICE
data << r.CONTRACTNO
data << r.DEALDATE
data << r.DEALTYPE
data << r.PAYTYPE
data << r.DEALPERIOD
data << r.IFELEVATOR
data << r.COMPAREAGARDEN
data << r.COMPAREATERRACE
data << r.COMPAREABALCONY
data << r.COMPAREABAYWINDOW
data << r.COMPAREAATTIC
data << r.COMPAREAGARAGE
data << r.COMPAREABASEMENT
data << r.COMPAREAOTHER
data << r.DATASOURCE
data << r.AGENCYID
data << r.AGENCYNAME
data << r.REMARK
data << r.CREATETIME1
data << r.UPDATETIME1
data << r.SOURCECODE
data << r.WORKDATE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
println "BIGDATA_DEAL_FJW 导出完成......"
}
def exportBigdataBuildingFJW(cityCode){
def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}'
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.CITYID
data << r.BUILDID
data << r.BUILDNO
data << r.BUILDNAME
data << r.BUILDALIAS
data << r.DISTID
data << r.COMMID
data << r.BUILDYEAR
if(r.BUILDTYPE){
def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",")
data << arrs[0].trim()
}else{
data << ''
}
data << r.BUILDSTRU
data << r.FLOORCNT
data << r.UNITCNT
data << r.HOUSECNT
data << r.IFELEVATOR
data << r.ELEVATORCNT
data << r.ELEVATORHOUSE
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
data << r.VALID_FLAG
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDING_FJW 上传Ftp完成......"
}
def exportEvaluateRate(){
def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW";
def allRows = []
sqlInstance.eachRow(sql,{r->
def data = []
data << r.ID
data << r.RATE_TYPE
data << r.RATE
data << r.RANK
data << r.CITY
data << r.REMARK
data << r.CREATETIME
data << r.UPDATETIME
data << r.SOURCECODE
allRows << data
});
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "EVALUATE_RATE_FJW 上传Ftp完成......"
}
def exportBigdataBuildprojFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......"
}
def exportBigdataCommlayoutpicFjw(){
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat"
}
FileUtils.batchCreateFile([], fileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",fileName, curDate)
println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......"
}
def exportOk(curDate){
// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd")
def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok"
if(CommonDatautils.isWindow()){
dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok"
}
FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false);
CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate)
}
}
@@ -920,7 +920,7 @@ class DataStandardBatch extends BaseBatch{
def findAllNoMatchData(){
DBObject qdata = new BasicDBObject();
qdata.put("s_date", SHORT_DATE)
qdata.put("city","福州")
// qdata.put("city","福州")
// qdata.put("source","安居客")
// qdata.put("name",Pattern.compile("龚家宅.*"))
qdata.put("status", new BasicDBObject(QueryOperators.EXISTS, false));
@@ -40,7 +40,7 @@ class DataStat {
def run(){
def source = "我爱我家"//
def source = "链家"//
def citys = findCrawlCity(source, 202209);
for(def city : citys){
def count1 = findCount(source, city, 202209);
@@ -26,7 +26,7 @@ class SynchDistrictPrice extends BaseBatch{
// "吉屋":"吉屋","365淘房":"365淘房","我爱我家":"我爱我家","诸葛找房APP":"诸葛找房","贝壳":"贝壳找房","链家":"链家","Q房":"Q房","麦田":"麦田",
// "中原地产":"中原地产","安居客小程序":"安居客","搜房":"搜房","乐有家":"乐有家","C21":"C21","城市房产":"城市房产","兔博士PC端":"兔博士"
def esSources = ["贝壳找房":"贝壳找房","链家":"链家","麦田":"麦田"]
def esSources = ["麦田":"麦田","链家":"链家"]
static main(args) {
@@ -42,7 +42,7 @@ class SynchDistrictPrice extends BaseBatch{
println "starting......"
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def citys = CommonUtils.findAllCity()
def citys = ["福州"]//CommonUtils.findAllCity()
esSources.each{esSource, mgSource->
for(def city : citys){