diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 6ef05438e..9ac398a0e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -23,7 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys_all; def rows = [] def headers = [] for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy index bd6ac75a5..e5272b8bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDistrictBatch.groovy @@ -10,6 +10,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql + /** * 建行模型估值项目 * 区域数据同步 @@ -17,33 +19,74 @@ import com.mongodb.DBCollection @BatchOwner(Watcher.GaoJuan) class BigdataDistrictBatch extends BaseBatch{ + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { runBatch(new BigdataDistrictBatch(), args); } @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; - def rows = [] - for(def city : citys){ - def datas = findDistrictCodeValue(city) - if(datas){ - rows.addAll(datas) - } - println city+" "+ datas.size() - } - - def allDatas = createDatas(rows) - - def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] - + def datas = readText("D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.txt") def rets = [] - rets << headers - rets.addAll(allDatas) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") - FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); + def index = 0 + for(def line : datas){ + def arrs = line[0].split("\\|@\\|") + def sql = "select count(*) as count from BIGDATA_DISTRICT_GJ where distid='${arrs[0]}'"; + def count = 0 + sqlInstance.eachRow(sql, { r -> + count = r.count + }) + if(count==0){ + rets << line + } + } + writeTxt(rets, "D:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ_add.txt") +// def citys = CcbCommonUtils.citys_all; +// def rows = [] +// for(def city : citys){ +// def datas = findDistrictCodeValue(city) +// if(datas){ +// rows.addAll(datas) +// } +// println city+" "+ datas.size() +// } +// +// def allDatas = createDatas(rows) +// +// def headers = ["行政区ID","行政区名称","行政区名称拼音","城市ID","国标码","显示顺序","行政区类别","备注","创建时间","更新时间","来源公司","逻辑行政区","逻辑行政区划代码","行政区划代码","数据时间"] +// +// def rets = [] +// rets << headers +// rets.addAll(allDatas) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\BIGDATA_DISTRICT_GJ.xlsx", rets, "数据") +// FileUtils.batchCreateFile(allDatas, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DISTRICT_GJ.dat",headers, "utf-8",false); } + + def writeTxt(def rows, def path){ + StringBuilder builder = new StringBuilder() + for(def row : rows){ + builder.append(row) + builder.append('\n') + } + org.apache.commons.io.FileUtils.writeStringToFile(new File(path), builder.toString(), "GBK") + } + private def readText(def fileName){ + File file = new File(fileName) + def ret = []; + //对文件内容的每一行进行分割处理,比较常用在处理csv文件 + file.splitEachLine(" ") { + def temp = it + ret << temp; + } + //转为Writable对象,可重定向输出 + file.asWritable() + return ret; + } + + def createDatas(rows){ def allRows = [] def index = 1 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index da1a37ff1..b8116db22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -53,62 +53,62 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // }) // } -// addBigdataDistrictFJW() + addBigdataDistrictFJW() - def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); - - def index = 0 - for (def city : citys) { - index++ - def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +// def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); // -// for(int i=0;i<10;i++){ -// if(districts.size()==0){ -// Thread.sleep(1000*60) -// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) -// }else{ -// break +// def index = 0 +// for (def city : citys) { +// index++ +// def districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +//// +//// for(int i=0;i<10;i++){ +//// if(districts.size()==0){ +//// Thread.sleep(1000*60) +//// districts = CcbCommonUtils.findAllDistrictsByOracle(cityCodeDataMap.get(city)) +//// }else{ +//// break +//// } +//// } +// +// def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) +// +// for(def dist : districts){ +// def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) +// if(obj){ +// if(obj.alias && obj.alias.size()>0){ +// dist.commalias = obj.alias.get(0) +// }else{ +// dist.commalias = "" +// } +// dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) +// dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) +// dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); +// dist.houseuse = "11" +// dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) +// dist.compid = new ObjectId() //物业公司id +// dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) // } // } - - def distMap = CommonDatautils.findAllDistricMaptByCity(city, "district", ["address","build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge"]) - - for(def dist : districts){ - def obj = distMap.get(dist.commid)//CommonDatautils.findDistrictFieldById(dist.commid,["build_structure","estate_type2","finish_building_date","alias","build_type","estate_charge","estate_type2"]) - if(obj){ - if(obj.alias && obj.alias.size()>0){ - dist.commalias = obj.alias.get(0) - }else{ - dist.commalias = "" - } - dist.buildstru = CcbCodeUtils.getBuildStructureCode(obj.build_structure) - dist.housetype = CcbCodeUtils.getDistHouseTypeCode(obj.estate_type2) - dist.buildyear = CcbCodeUtils.getBuildYear(obj.finish_building_date); - dist.houseuse = "11" - dist.buildtype = CcbCodeUtils.parseBuildType(dist.build_type) - dist.compid = new ObjectId() //物业公司id - dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) - } - } - println index +" "+city +" 小区加载完成 "+districts.size() - addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 1.小区表完成" - addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 2.小区建筑结构表" - addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 3.小区房屋类型表" - addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 4.小区建筑年代表" - addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 5.小区费用表" - addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 6.小区房屋用途表" - addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) - println index +" "+ city + " 7.地理位置表" - -// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 - - } +// println index +" "+city +" 小区加载完成 "+districts.size() +// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 1.小区表完成" +// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 2.小区建筑结构表" +// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 3.小区房屋类型表" +// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 4.小区建筑年代表" +// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 5.小区费用表" +// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 6.小区房屋用途表" +// addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) +// println index +" "+ city + " 7.地理位置表" +// +//// addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 +// +// } } def addBigdataCommunityGJRH(def districts, def cityid){