+37
@@ -50,6 +50,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
|
||||
"postgres", "postgres", "org.postgresql.Driver")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new BigdataDealBatch(), args);
|
||||
}
|
||||
@@ -106,6 +109,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
//
|
||||
// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
// println "txt 生成 " + rows.size()
|
||||
|
||||
saveToPgSql(rows)
|
||||
}
|
||||
|
||||
def createData(def rets, cityCodeDatas, distCodeMap, source){
|
||||
@@ -680,4 +685,36 @@ class BigdataDealBatch extends BaseBatch{
|
||||
latch.await()
|
||||
println city +" "+trades.size()+" "+new Date().toString()
|
||||
}
|
||||
|
||||
def saveToPgSql(rows){
|
||||
for(def r : rows) {
|
||||
for (def i = 0; i < r.size(); i++) {
|
||||
if("" == r[i]){
|
||||
r[i] = null
|
||||
}
|
||||
}
|
||||
r << null
|
||||
r << '2023-02-28'
|
||||
r << null
|
||||
r << '2999-12-31'
|
||||
r << null
|
||||
r << null
|
||||
r << null
|
||||
}
|
||||
|
||||
println "start: "+new Date()
|
||||
def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
|
||||
|
||||
sqlInstance.withBatch(20000, sql){ ps->
|
||||
for(def r : rows) {
|
||||
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
|
||||
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39],
|
||||
r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59],
|
||||
r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71])
|
||||
ps.toString()
|
||||
}
|
||||
}
|
||||
|
||||
println "end: "+new Date()
|
||||
}
|
||||
}
|
||||
|
||||
+57
-28
@@ -122,7 +122,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
totalCount.addAndGet(1)
|
||||
def commId = rMap.get("_id").toString()
|
||||
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2)
|
||||
|
||||
def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId)
|
||||
//def ret = findHousesByDistId(city, commId, "sell", startDate,endDate)
|
||||
// if(ret && ret.size()<100){//导历史房源需要,以后不需要
|
||||
// ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias)
|
||||
@@ -435,12 +436,24 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId){
|
||||
|
||||
def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2)
|
||||
def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1)
|
||||
rets1.addAll(rets2)
|
||||
return rets1;
|
||||
if(rets1){
|
||||
def newRets = []
|
||||
def houseIds = getAllHouseIds(commId)
|
||||
for(def r : rets1){
|
||||
if(!houseIds.contains(r.houseId)){
|
||||
newRets << r
|
||||
}else{
|
||||
println r.houseId +" 已存在"
|
||||
}
|
||||
}
|
||||
return newRets
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){
|
||||
@@ -474,8 +487,11 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
for (int k=0;k<houses.size();k++) {
|
||||
@@ -615,12 +631,20 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
def count = isExistFromHouse(seawater.id)
|
||||
if(count){
|
||||
println seawater.id +" 已存在"
|
||||
}else{
|
||||
rets << rowMap
|
||||
}
|
||||
rets << rowMap
|
||||
|
||||
// if(houseIds.contains(seawater.id)){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
// def count = isExistFromHouse(seawater.id)
|
||||
// if(count){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
@@ -667,9 +691,11 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
|
||||
def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"]
|
||||
def index = new Random().nextInt(6);
|
||||
|
||||
def rets = []
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder);
|
||||
|
||||
if(houses){
|
||||
|
||||
@@ -817,12 +843,20 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
rowMap.m_date = DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-18 10:00:00"//DateUtils.getConverDate(seawater.m_date)
|
||||
}
|
||||
|
||||
def count = isExistFromHouse(seawater.id)
|
||||
if(count){
|
||||
println seawater.id +" 已存在"
|
||||
}else{
|
||||
rets << rowMap
|
||||
}
|
||||
rets << rowMap
|
||||
|
||||
// if(houseIds.contains(seawater.id)){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
|
||||
// def count = isExistFromHouse(seawater.id)
|
||||
// if(count){
|
||||
// println seawater.id +" 已存在"
|
||||
// }else{
|
||||
// rets << rowMap
|
||||
// }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -978,12 +1012,6 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
|
||||
}
|
||||
|
||||
/*
|
||||
* 生成一个用不重复的ID
|
||||
*/
|
||||
def GenNonDuplicateID(randomLength){
|
||||
return Number(Math.random().toString().substr(3,randomLength) + Date.now()).toString(36)
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -1006,16 +1034,17 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
println ids.size()
|
||||
}
|
||||
|
||||
def getDistrictIds(def cityid){
|
||||
def sql = "SELECT commid FROM bigdata_forsale WHERE cityid = '"+cityid+"'";
|
||||
def getAllHouseIds(def commid){
|
||||
def sql = "SELECT FORSALEID FROM bigdata_forsale_gj WHERE commid = '"+commid+"'";
|
||||
|
||||
def distIds = new HashSet()
|
||||
def houseIds = new HashSet()
|
||||
sqlInstance.eachRow(sql,{it->
|
||||
distIds << it.commid
|
||||
houseIds << it.FORSALEID
|
||||
});
|
||||
return distIds;
|
||||
return houseIds;
|
||||
}
|
||||
|
||||
|
||||
def isExistFromHouse(def houseId){
|
||||
def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";
|
||||
|
||||
|
||||
Reference in New Issue
Block a user