This commit is contained in:
2021-12-28 17:49:56 +08:00
parent 5d796ad2e0
commit d7f3c0401a
3 changed files with 232 additions and 103 deletions
@@ -36,9 +36,9 @@ class BigdataBuildingBatch extends BaseBatch{
def distMap = getDistrictMap(city) def distMap = getDistrictMap(city)
def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap)
println city + " " + rets.size() println city + " " + rets.size()
def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" def fileName = "/home/goojia/ccb/bigdata_building_fjw.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat"
} }
FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true);
} }
@@ -24,30 +24,29 @@ class ExportDistrictRongHe extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
// def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.citys;
def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
// exportAllBigdataDealFJW() // exportAllBigdataDealFJW()
for(def city : citys) { for(def city : citys) {
def cityCode = cityCodeDataMap.get(city) def cityCode = cityCodeDataMap.get(city)
// exportBigdataCommunityGJRH(cityCode) exportBigdataCommunityGJRH(cityCode)
// exportBigdataBuildingFJW(cityCode) exportBigdataBuildingFJW(cityCode)
// exportBigdataCommfeeGj(cityCode) exportBigdataCommfeeGj(cityCode)
//
exportBigdataDealFJW(cityCode) exportBigdataDealFJW(cityCode, '2021-11-01')
// exportBigdataCommbuildstruGJ(cityCode) exportBigdataCommbuildstruGJ(cityCode)
// exportBigdataCommhousetypeGJ(cityCode) exportBigdataCommhousetypeGJ(cityCode)
// exportBigdataCommbuildyearGj(cityCode) exportBigdataCommbuildyearGj(cityCode)
// exportBigdataCommhouseuseGJ(cityCode) exportBigdataCommhouseuseGJ(cityCode)
// exportBigdataPositionGJ(cityCode) exportBigdataPositionGJ(cityCode)
// exportBigdataDistrictFJW(cityCode) exportBigdataDistrictFJW(cityCode)
exportBigdataAliasFJW(cityCode) exportBigdataAliasFJW(cityCode)
//
// exportBigdataBuildprojFjw(cityCode) exportBigdataBuildprojFjw(cityCode)
// exportBigdataCommlayoutpicFjw(cityCode) exportBigdataCommlayoutpicFjw(cityCode)
} }
// exportEvaluateRate() exportEvaluateRate()
} }
@@ -311,8 +310,25 @@ class ExportDistrictRongHe extends BaseBatch{
println "ADDRESS_ALIAS_FJW 上传Ftp完成......" println "ADDRESS_ALIAS_FJW 上传Ftp完成......"
} }
def exportBigdataDealFJW(cityCode){ def exportBigdataDealFJW(cityCode, dealdate){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('2021-06-01','yyyy-MM-dd')";
def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd")
def stop = false
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def index = 0
while(!stop){
index++
def dealStartStr = startDateStr
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat"
if(CommonDatautils.isWindow()){
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat"
}
def dealEndStr = DateUtils.add(startDateStr, 1)
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${dealStartStr}','yyyy-MM-dd') and dealdate<to_date('${dealEndStr}','yyyy-MM-dd')";
def allRows = [] def allRows = []
sqlInstance.eachRow(sql,{r-> sqlInstance.eachRow(sql,{r->
def data = [] def data = []
@@ -387,17 +403,16 @@ class ExportDistrictRongHe extends BaseBatch{
data << r.WORKDATE data << r.WORKDATE
allRows << data allRows << data
}); });
def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10"
def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(dealStartStr == endDateStr){
if(CommonDatautils.isWindow()){ stop = true
fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat"
} }
FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false);
CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}")
println "BIGDATA_DEAL_FJW 上传Ftp完成......" println "BIGDATA_DEAL_FJW 上传Ftp完成......${dealStartStr}"
startDateStr = dealEndStr
}
} }
def exportAllBigdataDealFJW(){ def exportAllBigdataDealFJW(){
def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ"; def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ";
@@ -0,0 +1,114 @@
package com.fangjia.script.gj
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.text.ParseException
import java.text.SimpleDateFormat
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class UpdateStatWeekend {
public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017";
public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", mongoDbUrl)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80)
static main(args) {
def task = new UpdateStatWeekend()
task.run()
}
def run(){
//"合肥","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆",
def citys = ["苏州"]
for(def city : citys){
def rets = findAddData(city)
process(city, rets)
}
}
def process(def city, def rets){
def index = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
for(def i=0;i<rets.size();i++){
final def r = rets[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
index.addAndGet(1)
updateProcess(r)
println index+"/"+rets.size() +" "+r.id
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def updateProcess(def r){
def curDate = DateUtils.parseFromTightYYYYMMDDhhmmss("2021-12-28 11:00:00")
if(r.m_date.getTime() < curDate.getTime()){
def weekendStr = DateUtils.formatDateByPattern(r.weekend, "yyyy-MM-dd")
weekendStr = DateUtils.add(weekendStr, -1)
def weekend = DateUtils.parseFromTightYYYYMMDD(weekendStr);
weekend = mongoDateTran(weekend)
DBObject data = new BasicDBObject()
data.put("weekend", weekend)
data.put("m_date", new Date())
try{
DIST_STAT_COLL.update(new BasicDBObject('_id', r.id), new BasicDBObject('$set', data))
}catch(Exception e){
DIST_STAT_COLL.remove(new BasicDBObject('_id', r.id))
println "[重复] "+ r.id
}
}
}
public static Date mongoDateTran(Date date) {
SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String dateStr = format.format(date);
format.setCalendar(new GregorianCalendar(new SimpleTimeZone(-57600000, "GMT")));
try {
return format.parse(dateStr);
} catch (ParseException e) {
return date;
}
}
def findAddData(city){
def q = new BasicDBObject()
q.put("city",city)
q.put("c_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-12-27")))
def dbcursor = DIST_STAT_COLL.find(q)
dbcursor.setOptions(Bytes.QUERYOPTION_NOTIMEOUT)
def rets = []
for(def d : dbcursor){
rets << ["id":d._id, "weekend":d.weekend,"c_date":d.c_date,"m_date":d.m_date]
if(rets.size()/10000==0){
print "."
}
}
return rets;
}
}