From eed2a3e44aa4446c90c5599680de5caa4870111f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 23 Oct 2023 20:41:00 +0800 Subject: [PATCH] =?UTF-8?q?7=E4=B8=AA=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/AddSzDealData.groovy | 16 ++- .../BigdataCommunityBatch.groovy | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 1 + .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 12 +- .../script/commData/CommonDatautils.groovy | 21 ++- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../fangjia/script/gj/poc/AddPriceTest.groovy | 123 +++++++++++++++++- .../fangjia/script/gj/xiamen/PoiWash.groovy | 103 ++++++++++++++- .../script/wenyan/temp/DongGuanBank.groovy | 6 +- 10 files changed, 266 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 6f83a19e0..a7b589cf9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳9月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -58,10 +58,16 @@ for(def d : datas){ }catch(Exception e){ println "room error" } - map.hall = arrs[1].toString().toInteger() - map.toilet = arrs[3].toString().toInteger() + if(arrs[1].toString()){ + map.hall = arrs[1].toString().toInteger() + } + if(arrs.size()==4 && arrs[3].toString()){ + map.toilet = arrs[3].toString().toInteger() + } }else{ - map.room = roomStr.toString().toDouble().toInteger() + if(roomStr.toString()){ + map.room = roomStr.toString().toDouble().toInteger() + } } } @@ -74,7 +80,7 @@ for(def d : datas){ if(trade_date){ map.trade_date = trade_date }else{ - def trade_date1 = DateUtils.getRandomDate("2023-08-01", "2023-08-31") + def trade_date1 = DateUtils.getRandomDate("2023-09-01", "2023-09-30") map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 49d4de707..c5e18a29f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -311,7 +311,7 @@ class BigdataCommunityBatch extends BaseBatch{ def sql = "insert into base.t0831_bigdata_community_gj_h values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)" def index = 0 - sqlInstance.withBatch(10000, sql){ ps-> + sqlInstance.withBatch(50000, sql){ ps-> for(def r : newRows) { index++ try{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 4139bce46..1e7e14b3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -63,6 +63,7 @@ class BigdataDealBatch extends BaseBatch{ //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) // def citys = CcbCommonUtils.citys; def citys = CcbCommonUtils.getRunningCitys(args) + citys = ['广州','重庆','南京','石家庄','廊坊','佛山','无锡','哈尔滨'] if(args && args.size()>=2){ curDate = args[1].toString().toInteger() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index ff83535fa..67a53ebc7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-08-25", "2023-09-27") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-09-27", "2023-10-24") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index df0d2d927..ec4ae67ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 - task.exportTrade()//导出未入库成交 + task.run()//统计最近3个月抓取的成交数据量 +// task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -93,13 +93,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys_all; def rets = [] - rets << ["城市","23年5月总量","23年6月总量","23年7月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","23年7月总量","23年8月总量","23年9月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线","我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" def startDate = "2023-05-01" - def date = "2023-06-01" - def endDate = "2023-07-01" - def newDate = "2023-08-01" + def date = "2023-08-01" + def endDate = "2023-09-01" + def newDate = "2023-10-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 728dd6c78..0ce543b28 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1740,14 +1740,14 @@ class CommonDatautils { } - public static def queryDistrictPrice(city, region, name, weekend){ + public static def queryFirstDistrictPrice(city, region, name){ DBObject queryCondition = new BasicDBObject() queryCondition.put("city", city); queryCondition.put("region", region); queryCondition.put("district_name", name); - queryCondition.put("weekend", weekend); +// queryCondition.put("weekend", weekend); - DBCursor cur = DIST_STAT_COLL.find(queryCondition) + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject("weekend", -1)).limit(1) cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def dbo = null; while(cur.hasNext()) { @@ -2215,6 +2215,21 @@ class CommonDatautils { } + public static def queryAllDistrictPrice(city, startDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put('weekend', startDate); + DBCursor cur = DIST_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def rMap = [:]; + while(cur.hasNext()) { + DBObject dbo = cur.next() + rMap.put(city + "##" + dbo.get("region")+"##"+ dbo.get("district_name"), dbo) + } + return rMap; + } + + public static def queryAllDistrictPrice(startDate){ DBObject queryCondition = new BasicDBObject() queryCondition.put('weekend', startDate); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index 488870b22..6899b1df8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-08-01" + static def statDate = "2023-09-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo - task.esfParser("D:\\数据\\友达\\2023年9月\\查询1.DBF");//友达二手房存入mongo +// task.esfParser("D:\\数据\\友达\\2023年10月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 43fb6f3b3..db5b55b54 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -9,8 +9,19 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import groovy.sql.Sql +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 价格对比,第三方,成交,建行价格,自已家价格 + */ class AddPriceTest { + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) + def pgsqlInstance = Sql.newInstance("jdbc:postgresql://192.168.0.236:5432/warehouse", "postgres", "postgres", "org.postgresql.Driver") @@ -26,10 +37,120 @@ class AddPriceTest { // addGXDPrice(filePath) // addCcbPrice(filePath) // addThirdPrice(filePath) - addFjwPrice(filePath) +// addFjwPrice(filePath) + exportPkPrice() +// addRegion() + } + def addRegion(){ + ExcelUtils.multiThreadingModify("D:\\数据\\上海农总行\\江苏省价格比对超20%的小区.xlsx",{d-> + def dist = CommonDatautils.findDistrictById(d['A']) + d['C'] = dist.region + }) + } + def exportPkPrice(){ + //"南京","苏州","无锡","扬州","常州","徐州","镇江","南通","泰州","连云港","盐城","淮安", + def citys = ["宿迁"] + + for(def c : citys){ + def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","alias"]) + println c+" district:"+districts.size() + + def fjwPriceMap =[:]// CommonDatautils.queryAllDistrictPrice(c, DateUtils.parseFromTightByPattern("2023-10-08","yyyy-MM-dd")) + def datas = addAllPrice(districts, fjwPriceMap) + ExcelUtils.write("D:\\数据\\上海农总行\\江苏省价格比对.xlsx", datas, c) + println c +" 完成" + } + } + + def addAllPrice(districts, fjwPriceMap){ + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def rets = new CopyOnWriteArrayList() + rets << ["id", "城市","区域","名称","建行价格","计算方式","可信度","房价网ToB价格","计算方式","可信度","近6个月成交均价"] + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + for(def i=0;i OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230701", "yyyyMMdd") +def endDate = DateUtil.parseDate("20230801", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -38,6 +38,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, "202301") +ExcelUtils.write("D:\\数据\\东莞\\东莞银行总行(不去重).xlsx", data, '202307')