This commit is contained in:
2023-02-17 15:12:07 +08:00
parent 1ccce2f38a
commit 72d565c760
3 changed files with 81 additions and 7 deletions
@@ -41,7 +41,6 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = CcbCommonUtils.getRunningCitys(args)
citys = ["上海"]
def rows = []
def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -251,7 +250,7 @@ class BigdataCommunityBatch extends BaseBatch{
sqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], null, '2023-02-28', null, '2999-12-31', null, null, null)
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}
}
@@ -57,9 +57,13 @@ class BigdataForsaleBatch extends BaseBatch{
public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"];
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60)
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/postgres?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new BigdataForsaleBatch(), args);
}
@@ -69,6 +73,7 @@ class BigdataForsaleBatch extends BaseBatch{
//目前查询库中是否已存在,用的是Oracle
def index = 0
def citys = CcbCommonUtils.getRunningCitys(args)
citys = ["福州"]
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
for(def city : citys){
index++
@@ -181,7 +186,9 @@ class BigdataForsaleBatch extends BaseBatch{
// headers = ["城市ID","出售ID","挂牌房屋ID","城市名称","行政区ID","行政区名称","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","单价","总价","出售付款方式","所在楼层","总楼层","建筑类型","室内结构","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","朝向","装修情况","是否有电梯","几梯几户","发布时间","到期时间","是否首发","房源数据来源","中介公司ID","中介公司名称","发布人","发布人电话","描述信息","备注","创建时间","更新时间","来源公司","附加信息"]
// }
FileUtils.batchCreateFile(allDatas, fileName, headers, "utf-8", true);
println "txt文件生成 "
println "txt文件生成->手动导入到Oracle "
saveToPgSql(allDatas)
}
@@ -610,9 +617,9 @@ class BigdataForsaleBatch extends BaseBatch{
def count = isExistFromHouse(seawater.id)
if(count){
rets << rowMap
}else{
println seawater.id +" 已存在"
}else{
rets << rowMap
}
}
@@ -812,9 +819,9 @@ class BigdataForsaleBatch extends BaseBatch{
def count = isExistFromHouse(seawater.id)
if(count){
rets << rowMap
}else{
println seawater.id +" 已存在"
}else{
rets << rowMap
}
}
}
@@ -1019,4 +1026,35 @@ class BigdataForsaleBatch extends BaseBatch{
return count;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_bigdata_forsale_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38], r[39], r[40],
r[41], r[42], r[43], r[44], r[45], r[46], r[47], r[48], r[49], r[50], r[51], r[52], r[53], r[54], r[55], r[56], r[57], r[58])
ps.toString()
}
}
println "end: "+new Date()
}
}
@@ -9,6 +9,7 @@ import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import groovy.sql.Sql
import org.bson.types.ObjectId
/**
@@ -18,6 +19,9 @@ import org.bson.types.ObjectId
@BatchOwner(Watcher.GaoJuan)
class StagingDistrictBatch extends BaseBatch{
def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse?stringtype=unspecified",
"postgres", "postgres", "org.postgresql.Driver")
static main(args) {
runBatch(new StagingDistrictBatch(), args);
}
@@ -47,6 +51,8 @@ class StagingDistrictBatch extends BaseBatch{
//ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据")
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
saveToPgSql(rows)
}
def createCityData(def dists, def fenQiMap){
@@ -86,4 +92,35 @@ class StagingDistrictBatch extends BaseBatch{
}
return fenQiMap;
}
def saveToPgSql(rows){
for(def r : rows) {
for (def i = 0; i < r.size(); i++) {
if("" == r[i]){
r[i] = null
}
}
r << null
r << '2023-02-28'
r << null
r << '2999-12-31'
r << null
r << null
r << null
}
println "start: "+new Date()
def sql = "insert into base.t0831_community_staging_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"
pgsqlInstance.withBatch(20000, sql){ ps->
for(def r : rows) {
ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20],
r[21], r[22], r[23], r[24], r[25], r[26], r[27], r[28], r[29], r[30], r[31], r[32], r[33], r[34], r[35], r[36], r[37], r[38])
ps.toString()
}
}
println "end: "+new Date()
}
}