diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 123ada0cf..0177411a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -41,7 +41,6 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) - citys = ["上海"] def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -251,7 +250,7 @@ class BigdataCommunityBatch extends BaseBatch{ sqlInstance.withBatch(20000, sql){ ps-> for(def r : rows) { ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], - r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], null, '2023-02-28', null, '2999-12-31', null, null, null) + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) ps.toString() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 7eac9b9f7..f90b1cfdc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -57,9 +57,13 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new BigdataForsaleBatch(), args); } @@ -69,6 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{ //目前查询库中是否已存在,用的是Oracle def index = 0 def citys = CcbCommonUtils.getRunningCitys(args) + citys = ["福州"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ index++ @@ -181,7 +186,9 @@ class BigdataForsaleBatch extends BaseBatch{ // headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"] // } FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true); - println "txt文件生成 " + println "txt文件生成->手动导入到Oracle " + + saveToPgSql(allDatas) } @@ -610,9 +617,9 @@ class BigdataForsaleBatch extends BaseBatch{ def count = isExistFromHouse(seawater.id) if(count){ - rets << rowMap - }else{ println seawater.id +" 已存在" + }else{ + rets << rowMap } } @@ -812,9 +819,9 @@ class BigdataForsaleBatch extends BaseBatch{ def count = isExistFromHouse(seawater.id) if(count){ - rets << rowMap - }else{ println seawater.id +" 已存在" + }else{ + rets << rowMap } } } @@ -1019,4 +1026,35 @@ class BigdataForsaleBatch extends BaseBatch{ return count; } + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_bigdata_forsale_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40], + r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58]) + ps.toString() + } + } + + println "end: "+new Date() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index ed95e4cc6..d11ca41ad 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +import groovy.sql.Sql import org.bson.types.ObjectId /** @@ -18,6 +19,9 @@ import org.bson.types.ObjectId @BatchOwner(Watcher.GaoJuan) class StagingDistrictBatch extends BaseBatch{ + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified", + "postgres", "postgres", "org.postgresql.Driver") + static main(args) { runBatch(new StagingDistrictBatch(), args); } @@ -47,6 +51,8 @@ class StagingDistrictBatch extends BaseBatch{ //ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + saveToPgSql(rows) + } def createCityData(def dists, def fenQiMap){ @@ -86,4 +92,35 @@ class StagingDistrictBatch extends BaseBatch{ } return fenQiMap; } + + + def saveToPgSql(rows){ + for(def r : rows) { + for (def i = 0; i < r.size(); i++) { + if("" == r[i]){ + r[i] = null + } + } + r << null + r << '2023-02-28' + r << null + r << '2999-12-31' + r << null + r << null + r << null + } + + println "start: "+new Date() + def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" + + pgsqlInstance.withBatch(20000, sql){ ps-> + for(def r : rows) { + ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], + r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38]) + ps.toString() + } + } + + println "end: "+new Date() + } }