diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index ba0985a47..3b9563cbe 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1837,6 +1837,7 @@ public class AddressController { Map params = new HashMap(); params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); params.put("address", model.getAddress()); + params.put("showCode", model.getShowCode()); try{ json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); } catch(Exception e){ diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java index e1048387a..ea10ab2f4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java @@ -22,6 +22,8 @@ public class AddesssMatchModel extends FangjiaApiModel { private String address_1; private String address_2; + private String showCode; + public String getAddress_1() { return address_1; } @@ -134,4 +136,12 @@ public class AddesssMatchModel extends FangjiaApiModel { public void setDebugName(String debugName) { this.debugName = debugName; } + + public String getShowCode() { + return showCode; + } + + public void setShowCode(String showCode) { + this.showCode = showCode; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 962d75c3b..f34d40421 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,10 +9,41 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳三月份成交数据.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳4月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) +//没有成交时间,随机生成 +//for(def d : datas){ +// def map = new BasicDBObject() +// map.city = "深圳" +// map.district_name = d['D'] +// map.region = d['B'] +// def roomStr = d['E'] +// if(roomStr){ +// def arrs = roomStr.split("-") +// try{ +// map.room = arrs[0].toString().toInteger() +// }catch(Exception e){ +// println "room error" +// } +// map.hall = arrs[1].toString().toInteger() +// map.toilet = arrs[3].toString().toInteger() +// } +// +// map.area = d['F'].toString().toDouble() +// map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) +// map.avg_price = (int)(d['H'].toString().toDouble()*10000) +// def trade_date = DateUtils.getRandomDate("2023-04-01", "2023-04-30") +// map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd") +// map.source = "深房家族" +// map.create_date = new Date() +// map.m_date = new Date() +// DEAL_PRICE_COLL.save(map) +//} + + + for(def d : datas){ def map = new BasicDBObject() map.city = "深圳" @@ -20,17 +51,15 @@ for(def d : datas){ map.region = d['B'] def roomStr = d['E'] if(roomStr){ - def arrs = roomStr.split("-") - map.room = arrs[0].toString().toInteger() - map.hall = arrs[1].toString().toInteger() - map.toilet = arrs[3].toString().toInteger() + map.room = roomStr.toString().toDouble().toInteger() } map.area = d['F'].toString().toDouble() map.total_price = (int)(d['G'].toString().toDouble().toInteger()*10000) - map.avg_price = (int)(d['H'].toString().toDouble()*10000) - def trade_date = DateUtils.getRandomDate("2023-03-01", "2023-03-31") - map.trade_date = DateUtils.parseFromTightByPattern(trade_date, "yyyy-MM-dd") + map.avg_price = (int)(d['H'].toString().toDouble()) + map.sell_price = (int)(d['I'].toString().toDouble()*10000) + def trade_date = d['J'] + map.trade_date = trade_date map.source = "深房家族" map.create_date = new Date() map.m_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 3418defb3..61bd39b7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -56,6 +56,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ FileUtils.batchCreateFile(rets, fileName,headers, "utf-8", false); println "txt 生成->导Oracle" + delPgData() saveToPgSql(rets) } @@ -110,6 +111,11 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ return rets; } + def delPgData(){ + sqlInstance.execute("delete from base.t0831_bigdata_commbuildtype_gj_h") + println "base.t0831_bigdata_commbuildtype_gj_h 数据已删除" + } + def saveToPgSql(rows){ def sqls = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 345491bd5..c52b21fcb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -56,8 +56,8 @@ class BigdataCommunityBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat","street","street_number"]) - def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 -// def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 +// def ccbMaps = CcbCommonUtils.findCcbDistrict(cityCodeDataMap.get(city)) //融合导数时使用 + def ccbMaps = CcbCommonUtils.findPgCcbDistrict(cityCodeDataMap.get(city)) //估值导数时使用 def rets = createCityDatas(cityCodeDataMap, distCodeMap, dists) rets = addDeleteDist(dists, ccbMaps, rets) rows.addAll(rets) @@ -78,10 +78,10 @@ class BigdataCommunityBatch extends BaseBatch{ println "txt 生成->导Oracle" -// delPgData() -// saveToPgSql(rows) - // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") + delPgData() + saveToPgSql(rows) +// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_community.xlsx", allDatas, "数据") // def datas = readTxt("d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat") // delPgData() // saveToPgSql2(datas) @@ -259,7 +259,7 @@ class BigdataCommunityBatch extends BaseBatch{ println "start: "+new Date() def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" - sqlInstance.withBatch(1, sql){ ps-> + sqlInstance.withBatch(50000, sql){ ps-> for(def r : rows) { try{ ps.addBatch(r[0], r[1], r[2], r[3], r[4], r[5], r[6], r[7], r[8], r[9], r[10], r[11], r[12], r[13], r[14], r[15], r[16], r[17], r[18], r[19], r[20], diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 5e7d45516..6854ae591 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -65,52 +65,51 @@ class BigdataDealBatch extends BaseBatch{ if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } - println "curDate:"+curDate - for(def city : citys){ -// if(!ccbCitys.contains(city)){ - addDealPriceMatch(city); -// } - } -// def rows = [] -// def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -// def index = 0 // for(def city : citys){ -// index ++ -// def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) -// def ppeDists = [:] -// -// def statMap = [:]//getDistrictStatByCity(city, weekend) -// if(city=="上海"){ -// def youDataTrade = getYouDaData(statMap) -// def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") -// rows.addAll(youdaDatas) -// println city +" "+youdaDatas.size() -// } -// -// def trades = getCityData(city, statMap, ppeDists) -// if(trades){ -// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") -// rows.addAll(rets) -// println "【${index}】"+city +" 成交共:"+rets.size() -// -// def cityDatas = [] -// cityDatas << headers -// cityDatas.addAll(rows) -// } -// +//// if(!ccbCitys.contains(city)){ +// addDealPriceMatch(city); +//// } // } -// -// def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" -// if(CommonDatautils.isWindow()){ -// fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" -// } -// -// FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); -// println "txt 生成 " + rows.size() -// -// saveToPgSql(rows) + def rows = [] + def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def index = 0 + for(def city : citys){ + index ++ + def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) + def ppeDists = [:] + + def statMap = [:]//getDistrictStatByCity(city, weekend) + if(city=="上海"){ + def youDataTrade = getYouDaData(statMap) + def youdaDatas = createData(youDataTrade, cityCodeDataMap, distCodeMap, "youda") + rows.addAll(youdaDatas) + println city +" "+youdaDatas.size() + } + + def trades = getCityData(city, statMap, ppeDists) + if(trades){ + def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") + rows.addAll(rets) + println "【${index}】"+city +" 成交共:"+rets.size() + + def cityDatas = [] + cityDatas << headers + cityDatas.addAll(rows) + } + + } + + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" + } + + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); + println "txt 生成 " + rows.size() + + saveToPgSql(rows) } def createData(def rets, cityCodeDatas, distCodeMap, source){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index db2c71d84..f4e661538 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -91,7 +91,7 @@ class BigdataForsaleBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230118, 20230219); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230318, 20230419, houseIdsfilter); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20230418, 20230519, houseIdsfilter); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) } @@ -203,7 +203,7 @@ class BigdataForsaleBatch extends BaseBatch{ } } - println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" size:"+ret.size()+" "+new Date().toLocaleString() + println totalCount.get() +" "+city+" "+rMap.region+" "+rMap.name+" 原始:"+ret.size()+" 过滤后:"+retMap.size() } catch (Exception e) { e.printStackTrace() @@ -1127,6 +1127,7 @@ class BigdataForsaleBatch extends BaseBatch{ return count; } + def saveToPgSql(rows){ for(def r : rows) { for (def i = 0; i < r.size(); i++) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index de1e2d965..d0a216521 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -430,8 +430,8 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); - q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 -// q.put("name","锦南浦三小区") +// q.put("c_date", new BasicDBObject('$lt', DateUtils.parseFromTightYYYYMMDD("2023-05-01")))//因5月1日新增过多,所以暂且先去掉 此批数据改为name_cat=13 +// q.put("name","金地玺湾") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index b78a31710..953a53a6b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-03-21", "2023-04-26") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-04-26", "2023-05-23") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) @@ -85,7 +85,7 @@ class GuiDingPriceBatch extends BaseBatch{ saveToPgSql(rows) //更新静态数据表 -// updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") + updateVersion(DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-28") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy index c0833369a..b2e287b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TestRepeat.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.datastore.driver.DSCollection +import com.fangjia.datastore.driver.DataStore +import com.mongodb.BasicDBObject import groovy.sql.Sql import java.util.concurrent.ConcurrentHashMap @@ -12,15 +15,24 @@ class TestRepeat { ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +// def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + private DSCollection seabird = new DataStore("192.168.0.100", 37017).getCollection("fangjia.seabird") static main(args) { def task = new TestRepeat() task.run() } def run(){ - + def q = new BasicDBObject() + q.put("cat", "sell") + q.put("city", "襄阳") + def count = seabird.count(q) + println count +// for(def d : dbcursor){ +// println d.name +// } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 84bbeaa2b..da8662b0e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202303//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202301 - def endDate = 202303 + def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));//202304// + def startDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -3, "yyyyMM")) //202302 + def endDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //202304 // def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3c0226e81..e8bd49824 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -92,13 +92,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年1月总量","23年2月总量","23年3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年2月总量","23年3月总量","23年4月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2023-01-01" - def date = "2023-02-01" - def endDate = "2023-03-01" - def newDate = "2023-04-01" + def startDate = "2023-02-01" + def date = "2023-03-01" + def endDate = "2023-04-01" + def newDate = "2023-05-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy index 024ad4123..640b024b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/UpdateData.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.FiledStrand import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import groovy.sql.Sql @@ -11,7 +13,26 @@ class UpdateData { static main(args) { def task = new UpdateData(); - task.run() +// task.run() + task.updateBlock() + } + + def updateBlock(){ + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def dists = CommonDatautils.findAllDistrictByCity(c, "district", ["city","region", "block","name"]) + for(def d : dists){ + if(d.block){ + def newBlock = FiledStrand.processBlock(d.city, d.region, d.block) + if(newBlock!= d.block){ + def udata = new BasicDBObject() + udata.put("block", newBlock) + println d.city+" "+d.block +" "+newBlock + DbUtils.seaweed.update(new BasicDBObject('_id',d._id),new BasicDBObject('$set', udata)) + } + } + } + } } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy new file mode 100644 index 000000000..58407d269 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/pgsql/ExportAllPgData.groovy @@ -0,0 +1,1040 @@ +package com.fangjia.script.ccbevaluation.pgsql + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils +import com.fangjia.script.ccbevaluation.CcbFtpUtils +import com.fangjia.script.ccbevaluation.ronghe.SSHHelperTest +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.jcraft.jsch.ChannelSftp +import groovy.sql.Sql +import org.apache.commons.codec.digest.DigestUtils + +class ExportAllPgData { + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", + "postgres", "postgres", "org.postgresql.Driver") + def citys = null + def cityCodeMap = CcbCommonUtils.findCityCodeValue() + +// def database = "base" + def database = "t_app_p9pha_plda." + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def linux_prefix = "/home/goojia/ccb_data/evaluation/" + def win_prefix = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\evaluation\\${curDate}\\" + + def testCitys = ['110000','130200','140200','150200','320500','320800','330100','420900','440100','610100'] + + static main(args) { + def task = new ExportAllPgData() + task.run(args) + } + + def run(args) { + citys = CcbCommonUtils.getRunningCitys(args) + def new_dists = [] + +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() +//// +//// //第三方网站权重表 +// exportWebsiteRate(); +// +// exportGuidedPrice()//随人工改变// +// exportCommunityGuidedPrice()//随政策// +// //处理小区情况 +// exportCommunityStaging()//没cityid +// exportBigdataCommunity()// +// +// exportCommunityBuildType()// +// + exportThirdPartyPrice2(new_dists)// + exportBigdataForsale(new_dists) //现在按updatetime导出 +// exportBigdataDeal(new_dists) //2022-07-01 + + + //计算结果相关 +// exportAvmEvaluationGJNB() +// exportAVMEvaluationItemGJNB() +// +// exportCityPrice() +// exportRegionPrice() + + exportCommunityBasePrice() +// exportCommunityBasePrice1(curMonth) + +// exportCommunityPdescribeGj() +// exportCommCaseDetailGj() +// +// exportInOk() +//// +// SSHHelperTest jschCommand = new SSHHelperTest(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_evaluation/output/eval_package.sh ${curDate}")); + } + + def exportEvaluationRateData() { + def sql = "select gujia_ceft_id,gujia_ceft_tp,gujia_bds_ceft,gujia_bds_seq, gujia_urbn,rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_evaluate_rate_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_ceft_id + data << r.gujia_ceft_tp + data << r.gujia_bds_ceft + data << r.gujia_bds_seq + data << r.gujia_urbn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + + + def fileName = linux_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("EVALUATE_RATE", allRows, fileName, curDate, false) + } + + def exportBigdataPoiData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_fsid,poi_nm,gujia_tp,gujia_tp_ecd,gujia_bds_ltt,gujia_bds_lgt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_bigdata_poi_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_fsid + data << r.poi_nm + data << r.gujia_tp + data << r.gujia_tp_ecd + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_POI_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_POI_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataDistrictData() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_admndiv_id,gujia_admndiv_nm,gujia_admndiv_nm_cpa,urbn_id,gujia_natstd_cd,gujia_dspl_seq,gujia_admndiv_cgy,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_lgc_admndiv_dsc,gujia_lgc_admndiv_cd,gujia_adiv_cd_val,gujia_bds_data_tm_val from "+database+".t0831_bigdata_district_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_admndiv_nm_cpa + data << r.urbn_id + data << r.gujia_natstd_cd + data << r.gujia_dspl_seq + data << r.gujia_admndiv_cgy + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_lgc_admndiv_dsc + data << r.gujia_lgc_admndiv_cd + data << r.gujia_adiv_cd_val + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DISTRICT_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DISTRICT_GJ", allRows, fileName, curDate, false) + } + + def exportWebsiteRate(){ + def sql = "select gujiadatawebst_src_id,gujia_data_webst_src,gujiadatawebstmpng_id,gujia_bds_wgt,gujia_bds_tp,gujia_src_co,urbn_id,gujia_urbn,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_website_config_gj_h"; + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujiadatawebst_src_id + data << r.gujia_data_webst_src + data << r.gujiadatawebstmpng_id + data << r.gujia_bds_wgt + data << r.gujia_bds_tp + data << r.gujia_src_co + data << r.urbn_id + data << r.gujia_urbn + data << r.data_crt_tm + data << r.data_udt_tm + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_WEBSITE_CONFIG_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("WEBSITE_CONFIG_GJ", allRows, fileName, curDate, false) + } + + + def exportGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id,comm_nm,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,to_char(gujia_drc_dt,'yyyy-mm-dd') as gujia_drc_dt,gujia_bds_drc_prc,gujia_add_psn,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_guided_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_drc_dt + data << r.gujia_bds_drc_prc + data << r.gujia_add_psn + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; + def fileName = linux_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_GUIDED_PRICE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("GUIDED_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityGuidedPrice() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_bds_id, comm_nm, urbn_id, urbn_nm, gujia_admndiv_id, gujia_admndiv_nm, gujia_bds_drc_prc, gujia_bds_drc_prc_src, rmrk_dsc, to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm, to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm, gujia_src_co from "+database+".t0831_community_guide_price_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_bds_id + data << r.comm_nm + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_drc_prc + data << r.gujia_bds_drc_prc_src + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_GUIDE_PRICE_GJ", allRows, fileName, curDate, false) + } + + def exportBigdataCommunity(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_admndiv_id,gujia_comm_id,gujia_admndiv_nm,comm_nm,gujia_dtl_adr,gujia_bds_ltt,gujia_bds_lgt,gujiabdsocpoldtotarea,gujiabds_bld_tot_area,gujia_bds_grnrate,gujia_bds_vol_rate,gujia_bds_park_num,gujia_bds_bld_yr_rc,gujia_bds_hs_use_val,totunits,gujia_bds_data_tm_val,gujiabds_udt_flag_val,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_comm_tp_val,gujia_comm_fmnm,gujia_str,gujia_hsno,gujiawthr_new_pan_ind,gujia_comm_intd,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,gujia_src_co,gujia_vlgtwn_str_mng,gujia_lgc_admndiv_cd,gujia_lgc_adiv,gujia_bds_bds_st from base.t0831_bigdata_community_gj_h where urbn_id='${cityCode}' ";//where cityId in (350200,150100) + pgsqlInstance.eachRow(sql,{r-> + def data = [] + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_comm_id + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_dtl_adr + data << r.gujia_bds_ltt + data << r.gujia_bds_lgt + data << r.gujiabdsocpoldtotarea + data << r.gujiabds_bld_tot_area + data << r.gujia_bds_grnrate + data << r.gujia_bds_vol_rate + data << r.gujia_bds_park_num + data << r.gujia_bds_bld_yr_rc + data << r.gujia_bds_hs_use_val + data << r.totunits + data << r.gujia_bds_data_tm_val + data << r.gujiabds_udt_flag_val + data << r.data_udt_tm + data << r.gujia_comm_tp_val + data << r.gujia_comm_fmnm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujiawthr_new_pan_ind + data << r.gujia_comm_intd + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.gujia_src_co + data << r.gujia_vlgtwn_str_mng + data << r.gujia_lgc_admndiv_cd + data << r.gujia_lgc_adiv + data << r.gujia_bds_bds_st + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMUNITY_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityStaging() { + + def allRows = [] + def sql = "select gujia_instm_id,gujia_comm_id,gujiacomm_prim_prd_id,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+".t0831_community_staging_gj_h"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_instm_id + data << r.gujia_comm_id + data << r.gujiacomm_prim_prd_id + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_STAGING_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("COMMUNITY_STAGING_GJ", allRows, fileName, curDate, false) + } + + def exportCommunityBuildType() { + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select urbn_id,gujia_comm_id,gujia_bld_tp_val,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_data_tm_val from "+database+".t0831_bigdata_commbuildtype_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_comm_id + data << r.gujia_bld_tp_val + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_data_tm_val + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_COMMBUILDTYPE_GJ_ALL_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_COMMBUILDTYPE_GJ", allRows, fileName, curDate, false) + } + + def exportThirdPartyPrice2(new_dists) { + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def allRows = [] + if(new_dists) { + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr} "; + def rets1 = getThirdPartyDatas(sql1) + allRows.addAll(rets1) + + def sql2 = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from "+database+".t0831_bigdata_third_party_price_gj_h where " + + "urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr} "; + def rets2 = getThirdPartyDatas(sql2) + allRows.addAll(rets2) + }else{ + + if(testCitys.contains(cityCode)){ + def sql = "select gujia_trdpt_avprc_id,urbn_id,gujia_admndiv_id,gujia_bds_id,comm_nm,gujia_dtsr_id,gujia_dtsr_dsc,gujiabds_avprc_dt_val,gujia_bds_comm_avprc,gujiabdscomm_hgst_prc,gujiabdscomm_lwst_prc,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm from base.t0831_bigdata_third_party_price_gj_h where urbn_id='${cityCode}' "; + allRows = getThirdPartyDatas(sql) + } + + } + if(allRows && allRows.size()>0){ + index = index + 1 + if(index==1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + } + } + + println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def getThirdPartyDatas(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.gujia_admndiv_id + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujia_dtsr_id + data << r.gujia_dtsr_dsc + data << r.gujiabds_avprc_dt_val + data << r.gujia_bds_comm_avprc + data << r.gujiabdscomm_hgst_prc + data << r.gujiabdscomm_lwst_prc + data << r.data_udt_tm + allRows << data + }) + return allRows; + } + + + def exportBigdataDeal(def new_dists) { + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}" + def rets1 = dealData(sql1) + + def sql2 = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id in ${newstr}" + def rets2 = dealData(sql2) + allRows.addAll(rets1) + allRows.addAll(rets2) + }else{ + def sql = "select urbn_id,gujia_bds_mdl_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_str,gujia_hsno,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_prd_val,gujia_bds_wbt_floor,gujia_lyr_tp_val,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_hs_use,gujia_indr_stc_val,gujia_twrd_val,gujia_dcrt_sttn,gujia_ases_bt_ln_num,gujia_bds_trdtckt_prc,gujia_bds_mdl_tot_prc,gujia_bds_dcl_tot_prc,gujiabds_ases_tot_prc,gujiasigntonlinectrid,to_char(txn_dt,'yyyy-mm-dd') as txn_dt,gujia_txn_tp_val,gujia_sell_pymtmod,gujia_bds_mdl_cyc,gujiawthrext_lift_ind,gujiabdslndfgrngvarea,gujiabdslutbl_gv_area,gujiabdssuntblgv_area,gujiabdsbayindowgarea,gujiabdstowerfrgvarea,gujia_bds_grg_gv_area,gujiabdsundgdrmgvarea,gujiabds_othr_gv_area,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_bds_mkdam,gujia_bds_data_tm_val " + + "from "+database+".t0831_bigdata_deal_gj_h where urbn_id='${cityCode}'"; + allRows.addAll(dealData(sql)) + } + } + + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + if (CommonDatautils.isWindow()) { + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_DEAL_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("BIGDATA_DEAL_GJ", allRows, fileName, curDate, false) + } + + + def dealData(def sql){ + def allRows = [] + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.urbn_id + data << r.gujia_bds_mdl_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_str + data << r.gujia_hsno + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_prd_val + data << r.gujia_bds_wbt_floor + data << r.gujia_lyr_tp_val + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_hs_use + data << r.gujia_indr_stc_val + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujia_ases_bt_ln_num + data << r.gujia_bds_trdtckt_prc + data << r.gujia_bds_mdl_tot_prc + data << r.gujia_bds_dcl_tot_prc + data << r.gujiabds_ases_tot_prc + data << r.gujiasigntonlinectrid + data << r.txn_dt + data << r.gujia_txn_tp_val + data << r.gujia_sell_pymtmod + data << r.gujia_bds_mdl_cyc + data << r.gujiawthrext_lift_ind + data << r.gujiabdslndfgrngvarea + data << r.gujiabdslutbl_gv_area + data << r.gujiabdssuntblgv_area + data << r.gujiabdsbayindowgarea + data << r.gujiabdstowerfrgvarea + data << r.gujia_bds_grg_gv_area + data << r.gujiabdsundgdrmgvarea + data << r.gujiabds_othr_gv_area + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_bds_mkdam + data << r.gujia_bds_data_tm_val + allRows << data + }); + return allRows + } + + + def exportBigdataForsale(new_dists){ + def fileName = linux_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + } + def index = 0 + for(def city : citys){ + def cityCode = cityCodeMap.get(city) + + def allRows = [] + if(new_dists){ + def newstr = "('"+new_dists.join("','")+"')" + def sql1 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' and gujia_admndiv_id not in ${newstr}"; + def rets1 = getHouseDatas(sql1) + allRows.addAll(rets1) + }else{ + if(testCitys.contains(cityCode)) { + def sql = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from base.t0831_bigdata_forsale_gj_h where urbn_id='${cityCode}' "; + allRows = getHouseDatas(sql) + } + } + println city +" "+allRows.size() + if(allRows.size()>0){ + index = index + 1 + if(index == 1){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + }else{ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + } + + } + } + + if(new_dists){//补充被修区域的数据 + def newstr = "('"+new_dists.join("','")+"')" + def sql2 = "select urbn_id,gujia_sell_id,gujia_listg_hs_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_comm_id,comm_nm,gujia_bld_id,bld_id,gujia_bld_nm,gujia_unit_id,unit_id,gujia_unit_nm,gujia_acc_id,gujia_dtl_adr,gujia_bds_bld_yr_rc,gujia_own_val,gujia_own_yrlmt_val,gujia_bds_unitprc,gujia_bds_tot_prc,gujia_sell_pymtmod,gujia_bds_wbt_floor,gujiabdstot_floor_num,gujia_bld_tp_val,gujia_indr_stc_val,gujia_bld_stc,guiabconstructionarea,gujiabdsset_innr_area,gujia_acc_tp_val,gujia_bds_room_num,gujia_bds_dept_num,gujia_bds_ktch_num,gujia_bds_hlt_btw_num,gujia_twrd_val,gujia_dcrt_sttn,gujiawthrext_lift_ind,gujiajiladderjiactval,to_char(anc_tm,'yyyy-mm-dd') as anc_tm,to_char(exdat,'yyyy-mm-dd') as exdat,gujia_wthr_strt,gujia_hs_src_dtsrc,intr_co_id,intr_co_nm,gujia_anc_psn,gujia_anc_psn_tel,gujia_dsc_inf_cntnt,rmrk_dsc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co,gujia_apd_inf " + + "from "+database+".t0831_bigdata_forsale_gj_h where gujia_admndiv_id in ${newstr}" + def rets2 = getHouseDatas(sql2) + if(rets2.size()>0){ + FileUtils.batchCreateFile(rets2, fileName, [], "utf-8", true); + } + } + println "BIGDATA_FORSALE_GJ 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "BIGDATA_FORSALE_GJ 上传Ftp完成......" + } + + def getHouseDatas(sql){ + def allRows = [] + def index = 0 + pgsqlInstance.eachRow(sql,{r-> + index = index+1 + def data = [] + data << r.urbn_id + data << r.gujia_sell_id + data << r.gujia_listg_hs_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_comm_id + data << r.comm_nm + data << r.gujia_bld_id + data << r.bld_id + data << r.gujia_bld_nm + data << r.gujia_unit_id + data << r.unit_id + data << r.gujia_unit_nm + data << r.gujia_acc_id + data << r.gujia_dtl_adr + data << r.gujia_bds_bld_yr_rc + data << r.gujia_own_val + data << r.gujia_own_yrlmt_val + data << r.gujia_bds_unitprc + data << r.gujia_bds_tot_prc + data << r.gujia_sell_pymtmod + data << r.gujia_bds_wbt_floor + data << r.gujiabdstot_floor_num + data << r.gujia_bld_tp_val + data << r.gujia_indr_stc_val + data << r.gujia_bld_stc + data << r.guiabconstructionarea + data << r.gujiabdsset_innr_area + data << r.gujia_acc_tp_val + data << r.gujia_bds_room_num + data << r.gujia_bds_dept_num + data << r.gujia_bds_ktch_num + data << r.gujia_bds_hlt_btw_num + data << r.gujia_twrd_val + data << r.gujia_dcrt_sttn + data << r.gujiawthrext_lift_ind + data << r.gujiajiladderjiactval + data << r.anc_tm + data << r.exdat + data << r.gujia_wthr_strt + data << r.gujia_hs_src_dtsrc + data << r.intr_co_id + data << r.intr_co_nm + data << r.gujia_anc_psn + data << r.gujia_anc_psn_tel + data << r.gujia_dsc_inf_cntnt + data << r.rmrk_dsc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + data << r.gujia_apd_inf + allRows << data +// if(allRows.size()%200000==0){ +// FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); +// allRows.clear() +// } + }); + return allRows + } + + + def exportAvmEvaluationGJNB(){ + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_bds_yrmo,gujia_bds_comm_num,gujiabdsvalmnycommnum,gujia_prc_cvr_rate,gujiabdsmnldpccommnum,gujia_mnl_drc_prc_pct,gujiabdssamp_comm_num,gujiaext_samp_clc_pct,gujiabdsnervetncomnum,gujianervethrunclcpct,gujiabds_tst_case_num,gujiabds_tst_comm_num,gujiatst_comm_num_pct,gujia_ppe15_cvr_rate,gujia_rto_mdn,gujia_rto_lv_san_ceft,gujiabdsmdltstcasenum,gujiabdsmdltstcommnum,gujiamdltstcommnumpct,gujiamdltst_ppe15_pct,gujiamdlrtolvsan_ceft,gujia_mdl_md_bit_rto,gujiabdstrdpttcasenum,gujiabdstrdpttcommnum,gujiatrdpttcommnumpct,gujiatrdpttstppe15pct,gujiatrdptrolvsanceft,gujiatrdpt_md_bit_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_evaluation_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_bds_yrmo + data << r.gujia_bds_comm_num + data << r.gujiabdsvalmnycommnum + data << r.gujia_prc_cvr_rate + data << r.gujiabdsmnldpccommnum + data << r.gujia_mnl_drc_prc_pct + data << r.gujiabdssamp_comm_num + data << r.gujiaext_samp_clc_pct + data << r.gujiabdsnervetncomnum + data << r.gujianervethrunclcpct + data << r.gujiabds_tst_case_num + data << r.gujiabds_tst_comm_num + data << r.gujiatst_comm_num_pct + data << r.gujia_ppe15_cvr_rate + data << r.gujia_rto_mdn + data << r.gujia_rto_lv_san_ceft + data << r.gujiabdsmdltstcasenum + data << r.gujiabdsmdltstcommnum + data << r.gujiamdltstcommnumpct + data << r.gujiamdltst_ppe15_pct + data << r.gujiamdlrtolvsan_ceft + data << r.gujia_mdl_md_bit_rto + data << r.gujiabdstrdpttcasenum + data << r.gujiabdstrdpttcommnum + data << r.gujiatrdpttcommnumpct + data << r.gujiatrdpttstppe15pct + data << r.gujiatrdptrolvsanceft + data << r.gujiatrdpt_md_bit_rto + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("AVM_EVALUATION_GJNB", allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_GJ", allRows, fileName1, curDate, false) + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + + def allRows = [] + + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,urbn_id,urbn_nm,gujia_rgon_id,gujia_rgon_nm,gujia_comm_id,gujia_comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_avprc,gujia_bds_clc_tp,gujia_bds_rlb,gujiabds_to_mkprc_mny,to_char(mdl_dt,'yyyy-mm-dd') as mdl_dt,gujiabds_valt_unitprc,gujia_area,gujia_dcrt_val,gujia_twrd_val,gujia_bds_floor_num,gujiabdstot_floor_num,gujia_data_tp,gujia_rto,gujia_mns_rto,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') AS data_crt_tm from "+database+"avm_evaluation_item_gj_h where urbn_id='${cityCode}'"; + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_rgon_id + data << r.gujia_rgon_nm + data << r.gujia_comm_id + data << r.gujia_comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_avprc + data << r.gujia_bds_clc_tp + data << r.gujia_bds_rlb + data << r.gujiabds_to_mkprc_mny + data << r.mdl_dt + data << r.gujiabds_valt_unitprc + data << r.gujia_area + data << r.gujia_dcrt_val + data << r.gujia_twrd_val + data << r.gujia_bds_floor_num + data << r.gujiabdstot_floor_num + data << r.gujia_data_tp + data << r.gujia_rto + data << r.gujia_mns_rto + data << r.data_crt_tm + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("AVM_EVALUATION_ITEM_GJNB",allRows, fileName, curDate, false) + createFileToFtp("AVM_EVALUATION_ITEM_GJ",allRows, fileName1, curDate, false) + } + + def exportCityPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_city_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + + def fileName = linux_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_CITY_PRICE_GJ_ADD_${curDate}_0001.dat" + } + createFileToFtp("CITY_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportRegionPrice(){ + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_trdpt_avprc_id,urbn_id,urbn_nm,gujia_admndiv_id,gxda_admndiv_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_bds_urbn_avprc,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_district_price_gj_h where urbn_id='${cityCode}'"; // + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_trdpt_avprc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gxda_admndiv_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_bds_urbn_avprc + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + }); + } + def fileName = linux_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_DISTRICT_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + createFileToFtp("DISTRICT_PRICE_GJ",allRows, fileName, curDate, false) + } + + def exportCommunityBasePrice(){ + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + for(def city : citys) { + def allRows = [] + def cityCode = cityCodeMap.get(city) + if(testCitys.contains(cityCode)){ + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";// + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + } + } + } + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + CcbFtpTestUtils.ftpUpload(null, fileName1, curDate) + } + + + def exportCommunityBasePrice1(yearMonth){ + + def fileName1 = linux_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName1 = win_prefix + "MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_bss_prc_id,urbn_id,urbn_nm,gujia_admndiv_id,gujia_admndiv_nm,gujia_bds_id,comm_nm,gujiabds_clc_yrmo_val,gujia_bds_clc_avprc,gujia_comm_basic_prc,gujia_bds_clmtd,gujiabds_fwd_samp_rto,gujia_bds_mns_grd,gujia_bds_cfdrg,gujiabdsarndcmprccrlt,gujia_bds_cfdc_dgr,to_char(data_crt_tm,'yyyy-mm-dd hh24:mi:ss') as data_crt_tm,to_char(data_udt_tm,'yyyy-mm-dd hh24:mi:ss') as data_udt_tm,gujia_src_co from "+database+"avm_community_base_price_gj_h where urbn_id='${cityCode}'";//gujiabds_clc_yrmo_val=${yearMonth} AND + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_bss_prc_id + data << r.urbn_id + data << r.urbn_nm + data << r.gujia_admndiv_id + data << r.gujia_admndiv_nm + data << r.gujia_bds_id + data << r.comm_nm + data << r.gujiabds_clc_yrmo_val + data << r.gujia_bds_clc_avprc + data << r.gujia_comm_basic_prc + data << r.gujia_bds_clmtd + data << r.gujiabds_fwd_samp_rto + data << r.gujia_bds_mns_grd + data << r.gujia_bds_cfdrg + data << r.gujiabdsarndcmprccrlt + data << r.gujia_bds_cfdc_dgr + data << r.data_crt_tm + data << r.data_udt_tm + data << r.gujia_src_co + allRows << data + + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + allRows.clear() + } + }); + } + + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName1, [], "utf-8", true); + } + } + + def exportCommunityPdescribeGj(){ + + def fileName = linux_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def allRows = [] + for(def city : citys) { + def cityCode = cityCodeMap.get(city) + def sql = "select gujia_fsid,gujia_comm_id,urbn_id,urbn_nm,admndiv_ecd,gujia_admndiv_nm,comm_nm,gujia_als_dsc,gujia_bds_bss_prc,gujiabds_cfdc_dgr_grd,gujiabdsrt3mlgcasenum,gujiabdslistgclwstprc,gujiabdslistgcaemdval,gujiabdslistgcase_avg,gujiabdslistgcemaxprc,gujia_bds_mo_val,gujiabdsavgwminvalavg,gujiabdsavgwmaxvalavg,gujia_bds_rel_err_val,gujia_dvtdgr_dsc,gujia_als_prc_rng,gujiaprcminandmax_rng,gujia_dsc,gujiabds_lmth_bss_prc,gujiabds_wthr_mnl_val,gujiabds_lmth_fnl_prc,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm,gujiabdsnervetuntwprc,gujia_prc_src,gujiabdsabtjlcmbitval,gujiabdsabtjlgcaseavg,gujia_indx_lbl,gujia_prim_pty_tp,gujia_comm_tp_val,gujiabds_ltst_mdl_prc,to_char(gujia_ltst_mdl_dt,'yyyy-mm-dd') as gujia_ltst_mdl_dt,gujiabdsidxnmo_moonmo,gujiabdsa1yiminmmonmo,gujiabdsidxnyr_moonmo,gujiabdsat1yspdgrscor,gujiabdsarnd1kscavprc from "+database+"avm_community_p_describe_gj_h where urbn_id='${cityCode}'"; + + pgsqlInstance.eachRow(sql, { r -> + def data = [] + data << r.gujia_fsid + data << r.gujia_comm_id + data << r.urbn_id + data << r.urbn_nm + data << r.admndiv_ecd + data << r.gujia_admndiv_nm + data << r.comm_nm + data << r.gujia_als_dsc + data << r.gujia_bds_bss_prc + data << r.gujiabds_cfdc_dgr_grd + data << r.gujiabdsrt3mlgcasenum + data << r.gujiabdslistgclwstprc + data << r.gujiabdslistgcaemdval + data << r.gujiabdslistgcase_avg + data << r.gujiabdslistgcemaxprc + data << r.gujia_bds_mo_val + data << r.gujiabdsavgwminvalavg + data << r.gujiabdsavgwmaxvalavg + data << r.gujia_bds_rel_err_val + data << r.gujia_dvtdgr_dsc + data << r.gujia_als_prc_rng + data << r.gujiaprcminandmax_rng + data << r.gujia_dsc + data << r.gujiabds_lmth_bss_prc + data << r.gujiabds_wthr_mnl_val + data << r.gujiabds_lmth_fnl_prc + data << r.data_crt_tm + data << r.gujiabdsnervetuntwprc + data << r.gujia_prc_src + data << r.gujiabdsabtjlcmbitval + data << r.gujiabdsabtjlgcaseavg + data << r.gujia_indx_lbl + data << r.gujia_prim_pty_tp + data << r.gujia_comm_tp_val + data << r.gujiabds_ltst_mdl_prc + data << r.gujia_ltst_mdl_dt + data << r.gujiabdsidxnmo_moonmo + data << r.gujiabdsa1yiminmmonmo + data << r.gujiabdsidxnyr_moonmo + data << r.gujiabdsat1yspdgrscor + data << r.gujiabdsarnd1kscavprc + allRows << data + }); + } + createFileToFtp("community_p_describe_gj",allRows, fileName, curDate, false) + } + + def exportCommCaseDetailGj(){ + def fileName = linux_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } +// for(def city : citys){ +// def cityCode = cityCodeMap.get(city) +// def sql = "select gujia_fsid, urbn_id, gujia_comm_id, gujia_case_id,gujia_bds_crterm,to_char(data_crt_tm,'yyyy-mm-dd') as data_crt_tm from "+database+"avm_comm_case_detail_gj_h where urbn_id='${cityCode}' and gujia_bds_crterm=${month}"; +// def allRows = [] +// pgsqlInstance.eachRow(sql,{r-> +// def data = [] +// data << r.gujia_fsid +// data << r.urbn_id +// data << r.gujia_comm_id +// data << r.gujia_case_id +// data << r.gujia_bds_crterm +// data << r.data_crt_tm +// allRows << data +//// if(allRows.size()%200000==0){ +//// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", true); +//// allRows.clear() +//// } +// }); +// +// FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); +// } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + } + + def exportMd5(){ + Vector files = CcbFtpUtils.listFileNames(curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +":"+ DigestUtils.md5Hex(file.getFilename()) + } + } + def fileName = linux_prefix + "md5.txt" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "md5.txt" + } + createFileToFtp("md5.txt",fileRows, fileName, curDate, false) + } + + + def exportInOk(){ + Vector files = CcbFtpTestUtils.listFileNames(null, curDate) + Iterator fileVec = files.iterator() + def fileRows = [] + while(fileVec.hasNext()){ + ChannelSftp.LsEntry file = fileVec.next() + + if(org.apache.commons.lang.StringUtils.isNotBlank(file.getFilename()) && file.getFilename().endsWith(".dat")){ + fileRows << file.getFilename() +" "+CcbFtpTestUtils.getFileLength(null, file.getFilename(), curDate) + } + } + def fileName = linux_prefix + "/${curDate}.ok" + if(CommonDatautils.isWindow()){ + fileName = win_prefix + "${curDate}.ok" + } + + FileUtils.createTxtFile(fileRows, fileName, "utf-8", false); + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println "ok文件上传Ftp完成......" + } + + + def createFileToFtp(tableName,allRows, fileName, curDate, isAppend){ + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", isAppend); + println tableName+" 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload(null, fileName, curDate) + println tableName+" 上传Ftp完成......" + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 43132328a..852f3783a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -39,6 +39,7 @@ class ExportDistrictRongHe extends BaseBatch{ citys = CcbCommonUtils.getRunningCitys(args) def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") def new_dists = []//["350604","350605","410307","410308","450181"] @@ -66,9 +67,6 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCityPrice(queryDate) exportBigdataDistPrice(queryDate) exportBigdataCommPrice(queryDate) - - - //1月10日 新增5张表,导92城,增量数据,2月10日导344城,其中252城所有历史数据,92城增量,3月后可能不用提供数据 // exportCommunityBasePrice(lastMonth) // exportCommunityPdescribeGj(lastMonth) @@ -81,15 +79,15 @@ class ExportDistrictRongHe extends BaseBatch{ } if(curDate.toString().endsWith("28")){ - exportPgBigdataSupplierStatistics(lastMonth) + exportPgBigdataSupplierStatistics(curMonth) //因为融合暂不切库,所以需要加传一张第三方均价表到融合中, 从1月10日开始传输,长期供 - def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def thirdPartyMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // exportThirdPartyPrice2(thirdPartyMonth, new_dists) exportPgThirdPartyPrice2(thirdPartyMonth, new_dists) - exportPgCommunityBasePrice(lastMonth) - exportPgCommunityPdescribeGj(lastMonth) - exportPgBigdataModelingBase(lastMonth) - exportPgGuideStatistics(lastMonth) + exportPgCommunityBasePrice(curMonth) + exportPgCommunityPdescribeGj(curMonth) + exportPgBigdataModelingBase(curMonth) + exportPgGuideStatistics(curMonth) } SSHHelper jschCommand = new SSHHelper(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy index 7447a6712..d008b0f48 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/dataAna/CityInfoAnaBatch.groovy @@ -69,27 +69,24 @@ class CityInfoAnaBatch extends BaseBatch { alias:'alias',addr_alias:'addrAlias','main_type_area':'mainTypeArea',loudong_count:'loudongCount',car_site_count:'carSiteCount',mapping_alias:'mappingAlias',is_elevator:'elevator', township:'township',kindergartens:'kindergartens',main_type:'mainType',land_property:'landProperty',usage_card:'usageCard',occupancy_rate:'occupancyRate',land_usage_start:'landUsageStart' ] + //out_pics : 'outPics', pics : 'pics',household_count :'householdCount',build_structure :'buildStructure', facilities : 'facilities',characteristic:'characteristic',licence:'licence',sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany', +//conditioner_time:'conditionerTime',network_way:'networkWay',fitment_level:'fitmentLevel',lighting_ventilation:'lightingVentilation', public static final def OFFICE_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',household_count :'householdCount',open_date :'openDate',live_date :'liveDate', + name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',cubage_percent :'cubagePercent',virescence_percent :'virescencePercent', - car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate',build_structure :'buildStructure', - facilities : 'facilities',characteristic:'characteristic',car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics',addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo',licence:'licence', - sideset:'sideset',real_use_percent:'realUsePercent',build_designer:'buildDesigner',cons_company:'consCompany',estate_charge:'estateCharge', - floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife', - conditioner_time:'conditionerTime',elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount', - goods_elevator_count:'goodsElevatorCount',network_way:'networkWay',fitment_level:'fitmentLevel',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea', - is_foreign:'foreign',is_register:'register',lighting_ventilation:'lightingVentilation',lobby_height:'lobbyHeight' + car_site :'carSite',estate_charge :'estateCharge',finish_building_date :'finishBuildingDate',start_building_date :'startBuildingDate', + car_site_count :'carSiteCount',bed_kind:'bedKind',alias:'alias',lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', addr_alias:'addrAlias',business_quarter:'businessQuarter',loop_line_info:'loopLineInfo', + estate_charge:'estateCharge',floor_info:'floorInfo',underground_car_site:'undergroundCarSite',land_car_site:'landCarSite',parking_fee:'parkingFee',sight:'sight',property_life:'propertyLife', + elevators_count:'elevatorsCount',person_elevators_count:'personElevatorsCount',goods_elevator_count:'goodsElevatorCount',standard_height:'standardHeight',clear_height:'clearHeight',standard_area:'standardArea', + is_foreign:'foreign',is_register:'register',lobby_height:'lobbyHeight',township:'township',divisible :'divisible',land_floor:'landFloor',underground_floor:'undergroundFloor' ] + // out_pics : 'outPics', pics : 'pics', public static final def SHOP_FIELDS_MAP = [ - name :'name',region :'region',block :'block',address :'address',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', - all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany', - rent_type :'rentType',lease_term :'leaseTerm',car_site :'carSite',estate_charge :'estateCharge',divisible :'divisible',floor :'floor', - finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate', - bay_area :'bayArea', property_life :'propertyLife',unified_manage :'unifiedManage',licence :'licence',business_status :'businessStatus', - lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2', out_pics : 'outPics', pics : 'pics' - + name :'name',region :'region',block :'block',address :'address', alias:'alias',addr_alias:'addrAlias',estate_type2 :'estateType2',open_date :'openDate',live_date :'liveDate', + all_area :'allArea',build_area :'buildArea',build_type :'buildType',developer :'developer',estate_company :'estateCompany',car_site :'carSite',estate_charge :'estateCharge',floor :'floor', + finish_building_date :'finishBuildingDate',business_quarter :'businessQuarter',start_building_date :'startBuildingDate',bay_area :'bayArea', property_life :'propertyLife',business_status :'businessStatus',parking_fee:'parkingFee', + lat : 'lat',lng : 'lng',lat2 : 'lat2',lng2 : 'lng2',township:'township',loop_line_info :'loopLineInfo',land_floor:'landFloor',underground_floor:'undergroundFloor',real_use_percent:'realUsePercent' ] public static CITY_GRADE_MAP = [1000: "一线" , 1500 : "新一线" , 2000 : "二线" , 3000:"三线" ,4000 : "四线" ,5000 : "五线"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy index e6cdef564..1b7d3b371 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FiledStrand.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.gj +import com.fangjia.core.BlockData +import com.fangjia.core.RegionData import com.fangjia.core.util.DateUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.linxsh.util.AddressUtil @@ -337,11 +339,12 @@ class FiledStrand { return null } - static def processBlock(def block){ + static def processBlock(city, region, def block){ if(block){ if(block.contains("无") || block.toString().length()>10){ return null } +// block = BlockData.formatBlock(city, region, block); return block } return null diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 0842f9b06..16cae9383 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-03-01" + static def statDate = "2023-04-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年3月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年5月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy index 30a4e51ab..78270dc6c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/FileUtils.groovy @@ -83,7 +83,7 @@ class FileUtils { def f = new File(file) def ret = []; if(f.exists()){ - def lines = org.apache.commons.io.FileUtils.readLines(f) + def lines = org.apache.commons.io.FileUtils.readLines(f,"utf-8") for(line in lines) { ret << line; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 2806a48f1..a04bae008 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -67,7 +67,7 @@ class DistrictFieldAddBatch extends BaseBatch{ @Override void process(Object args) { - def citys = CommonDatautils.findAllCity().reverse() + def citys = ["漳州"]//CommonDatautils.findAllCity().reverse() def analysRets = [] @@ -279,7 +279,7 @@ class DistrictFieldAddBatch extends BaseBatch{ newVal = FiledStrand.processAddress(esVal, dist.city) } if(seaweedfield =="block"){ - newVal = FiledStrand.processBlock(esVal) + newVal = FiledStrand.processBlock(dist.city, dist.region, esVal) } if(seaweedfield == "build_type"){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 319e44ef4..a59b0747a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -26,8 +26,10 @@ class SeaweedDataMatchProcess { rMap.region = obj.region rMap.alias = obj.alias rMap.name = obj.name - rMap.lat = obj.lat/10000000000.0 - rMap.lng = obj.lng/10000000000.0 + if(obj.lat){ + rMap.lat = obj.lat/10000000000.0 + rMap.lng = obj.lng/10000000000.0 + } rMap.id = obj._id.toString() rMap.address = obj.address } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index f71b47a56..4961abd56 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.paimai import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils @@ -34,6 +35,7 @@ public class BeiKeCrawlByChrome { DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) def startMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + //广州,佛山 重庆 static main(args) { def task = new BeiKeCrawlByChrome(); task.run() @@ -49,27 +51,8 @@ public class BeiKeCrawlByChrome { WebDriver driver = new ChromeDriver() def citys = getAllCityUrls(driver) -// citys = citys.reverse() -// def index = 0 -// for(def c : citys){ -// index++ -// if(c.city=="合肥"){ -// break -// } -// } -// citys = citys[index..citys.size()-1] - def start = 0 -// for(def i=0;i登录"))){ - driver = getLogin(driver) + driver = getLogin(driver, url) driver.get(url) Thread.sleep(5000) s = driver.getPageSource() @@ -127,8 +110,14 @@ public class BeiKeCrawlByChrome { def crawlData(driver, citys){ def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + def runedCitys = FileUtils.readTxt("d:\\数据\\成交抓取\\贝壳已抓城市.txt") + for(def m : citys){ def city = m[0] + if(runedCitys.contains(city)){ + println city +" 已运行" + continue + } def url = m[1] def newUrl = url+"/chengjiao/" driver.get(newUrl) @@ -145,10 +134,11 @@ public class BeiKeCrawlByChrome { println url +" 详情链接:"+detailUrls.size() } } - + FileUtils.batchCreateFile([[city]], "d:\\数据\\成交抓取\\贝壳已抓城市.txt", [], "utf-8", true); // def detailUrls = getPageDeail(driver, m.city, m.url) // println m.city +" 详情链接:"+detailUrls.size() // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + } } @@ -434,27 +424,32 @@ public class BeiKeCrawlByChrome { return list } - def getLogin(driver){ + def getLogin(driver, url){ // WebDriver driver = new ChromeDriver() - driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); + def root = url.toString().substring(url.toString().indexOf("//")+2, url.toString().indexOf(".com")+4) + driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252F"+root+"%252Fchengjiao%252F"); WebDriverWait wait = new WebDriverWait(driver,20) Thread.sleep(10000) - wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + try{ + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); - WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 - WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 - WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - elemUsername.sendKeys("15021384995"); + elemUsername.sendKeys("15021384995"); - elemPassword.sendKeys("12qwaszx"); + elemPassword.sendKeys("12qwaszx"); - btn.submit(); // 点击行为; - Thread.sleep(10000) + btn.submit(); // 点击行为; + Thread.sleep(10000) + }catch(Exception e){ + Thread.sleep(10000) + } return driver } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index a2e8be163..aaf0c9c0d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -118,6 +118,7 @@ public class LianjiaCrawlByChrome { } def run(){ + System.setProperty("webdriver.chrome.driver", "D:\\Program Files\\chrome-driver\\chromedriver.exe"); System.setProperty("http.nonProxyHosts", "localhost"); WebDriver driver = new ChromeDriver() @@ -523,24 +524,21 @@ public class LianjiaCrawlByChrome { Thread.sleep(10000) try{ wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + + elemUsername.sendKeys("15021384995"); + + elemPassword.sendKeys("12qwaszx"); + + btn.submit(); // 点击行为; + Thread.sleep(10000) }catch(Exception e){ - Thread.sleep(60000) - getLogin(driver, url) + Thread.sleep(10000) } - - WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 - - WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 - - WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 - - - elemUsername.sendKeys("15021384995"); - - elemPassword.sendKeys("12qwaszx"); - - btn.submit(); // 点击行为; - Thread.sleep(20000) return driver } }