diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index fe8a32a3c..d2cbed0d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -50,6 +50,9 @@ class BigdataDealBatch extends BaseBatch{ def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataDealBatch(), args); } @@ -106,6 +109,8 @@ class BigdataDealBatch extends BaseBatch{ // // FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); // println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ @@ -680,4 +685,36 @@ class BigdataDealBatch extends BaseBatch{ latch.await() println city +" "+trades.size()+" "+new Date().toString() } + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_deal_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + sqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], + r[40], r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49],r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58], r[59], + r[60], r[61], r[62], r[63], r[64], r[65], r[66], r[67], r[68], r[69],r[70], r[71]) + ps.toString() + } + } + + println "end: "+new Date() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index f90b1cfdc..0f85fe67b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -122,7 +122,8 @@ class BigdataForsaleBatch extends BaseBatch{ totalCount.addAndGet(1) def commId = rMap.get("_id").toString() - def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2) + + def ret = findHouses(city, rMap.region, "sell", startDate,endDate, rMap.name, rMap.alias, rMap.estate_type2, commId) //def ret = findHousesByDistId(city, commId, "sell", startDate,endDate) // if(ret && ret.size()<100){//导历史房源需要,以后不需要 // ret = findHouses(city, rMap.region, "sell", 20190701, endDate, rMap.name, rMap.alias) @@ -435,12 +436,24 @@ class BigdataForsaleBatch extends BaseBatch{ } - def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2, def commId){ def rets1 = findHousesByPdate(city, region, cat, startDate, endDate, name, alias, estate_type2) def rets2 = findHousesByMdate(city, region, cat, startDate, endDate, name, alias, estate_type2, rets1) rets1.addAll(rets2) - return rets1; + if(rets1){ + def newRets = [] + def houseIds = getAllHouseIds(commId) + for(def r : rets1){ + if(!houseIds.contains(r.houseId)){ + newRets << r + }else{ + println r.houseId +" 已存在" + } + } + return newRets + } + return []; } def findHousesByPdate(def city, def region, def cat, def startDate, def endDate, def name, def alias, def estate_type2){ @@ -474,8 +487,11 @@ class BigdataForsaleBatch extends BaseBatch{ } + def hosts = ["192.168.0.206","192.168.0.211","192.168.0.212","192.168.0.213","192.168.0.214","192.168.0.215"] + def index = new Random().nextInt(6); + def rets = [] - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def houses = ElasticUtils.executeSimpleQuery(hosts[index], "seawater-20*", queryBuilder); if(houses){ for (int k=0;k - distIds << it.commid + houseIds << it.FORSALEID }); - return distIds; + return houseIds; } + def isExistFromHouse(def houseId){ def sql = "SELECT count(FORSALEID) as count FROM bigdata_forsale_gj WHERE FORSALEID = '"+houseId+"'";