From e8d12518705091f1749d85aee1b7b5cfdf75889d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Nov 2022 17:54:00 +0800 Subject: [PATCH 1/2] update --- .../ccbevaluation/BigdataBuildingBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 5 ++ .../ccbevaluation/GuiDingPriceBatch.groovy | 7 ++- .../fangjia/script/ccbevaluation/Test.groovy | 60 +++++++++---------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- .../ronghe/BigdataCommunityGJRHBatch.groovy | 10 +++- .../gj/paimai/BeiKeCrawlByChrome.groovy | 2 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- 8 files changed, 52 insertions(+), 38 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 7500406cc..431972b91 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -36,7 +36,7 @@ class BigdataBuildingBatch extends BaseBatch{ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap) println city + " " + rets.size() - def fileName = "/home/goojia/ccb/bigdata_building_fjw.dat" + def fileName = "/ccb/bigdata_building_fjw.dat" if(CommonDatautils.isWindow()){ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index d9d7660c0..909ea8025 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -562,4 +562,9 @@ class CcbCommonUtils { return maxSeqno } + public static def deleteGuidedPrice(def dealdate){ + def sql = "delete from guided_price_gj where dealdate=to_date('${dealdate}','yyyy-mm-dd')" + sqlInstance.executeUpdate(sql) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 66e8081dd..bd64cc3f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -30,6 +30,8 @@ class GuiDingPriceBatch extends BaseBatch{ def sDate= Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); def static final DBCollection dataProcessColl = CrawalDBUtil.getDataProcessCol(); + def curDate= DateUtils.formatDateByPattern(new Date(), "yyyy-MM")+"-15"; + static main(args) { runBatch(new GuiDingPriceBatch(), args); } @@ -39,12 +41,13 @@ class GuiDingPriceBatch extends BaseBatch{ def citys = CcbCommonUtils.getRunningCitys(args) def rows = [] + CcbCommonUtils.deleteGuidedPrice(curDate) for (def city : citys) { def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-11") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-10-27", "2022-11-18") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) @@ -150,7 +153,7 @@ class GuiDingPriceBatch extends BaseBatch{ ret << CcbCommonUtils.getCcbCity(dist.city) ret << areaData.district_code ret << areaData.district - ret << DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") + ret << curDate//DateUtils.formatDateByPattern(ps.guiding_date, "yyyy-MM-dd HH:mm:ss") ret << ps.guiding_value ret << "房价网" ret << "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index f8e04a350..d36f08cc1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -447,23 +447,23 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business ////},0) // -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\11.11\\删除小区(2).xlsx",{d-> -// if(d['A']){ -// def dbo = CommonDatautils.findDistrictById(d['C']) -// if(dbo){ -// def udata = new BasicDBObject() +ExcelUtils.multiThreadingModify("D:\\数据\\建行\\二期\\252个城市删除小区.xlsx",{d-> + if(d['A']){ + def dbo = CommonDatautils.findDistrictById(d['D']) + if(dbo){ + def udata = new BasicDBObject() // udata.put("name_cat", 11) -// udata.put("status", 1) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", dbo._id) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// }else{ -// println dbo._id.toString() +" 没找到" -// } -// } -//},0) + udata.put("status", 1) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dbo._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println dbo._id.toString() +" 没找到" + } + } +},0) //def citys = ["海口","三亚","三沙","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水","保亭","琼中"] //for(def c : citys){ @@ -471,20 +471,20 @@ def DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business // println c +" "+districts.size() //} -def cityCodeMap = CcbCommonUtils.findCityCodeValue() -ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d -> -// cityCodeMap.each {k,v-> -// if(d['A'].toString()==v){ -// d['C'] = k -// println k -// } -// } -// if(!d['D']){ -// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) -// } - d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) - -}); +//def cityCodeMap = CcbCommonUtils.findCityCodeValue() +//ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\252城第三方均价.xlsx", { d -> +//// cityCodeMap.each {k,v-> +//// if(d['A'].toString()==v){ +//// d['C'] = k +//// println k +//// } +//// } +//// if(!d['D']){ +//// d['D'] = findCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) +//// } +// d['F'] = findThirdCommCount(d['A'].toString().toDouble().toInteger().toString(), sqlInstance94) +// +//}); //def citys = CcbCommonUtils.citys_252 //def runCitys = ['聊城','宝鸡','韶关','阜阳','宜春','周口','河源','忻州','三门峡','永州','衢州','盘锦','白银','玉溪','达州','景德镇','鹤壁','辽阳','六盘水','张家界','海西州','兴安盟','阿坝州','那曲','西双版纳州','辽源','邯郸','平顶山','南阳','商丘','南平','黄冈','鄂尔多斯','泸州','六安','葫芦岛','潮州','宿州','庆阳','怀化','张掖','黔东南','云浮','本溪','百色','伊春','昭通','锡林郭勒盟','克拉玛依','天门','博尔塔拉','大兴安岭','潍坊','绵阳','梅州','襄阳','包头','枣庄','东营','信阳','晋中','开封','承德','茂名','荆门','红河','陇南','铁岭','黄山','咸宁','延安','广元','乌兰察布','随州','益阳','阳泉','内江','酒泉','毕节','安顺','丽江','商洛','来宾','黔西南','防城港','铜仁','儋州','崇左','衡阳','咸阳','新乡','南充','荆州','德阳','汉中','湘潭','赤峰','佳木斯','丹东','淮南','抚州','丽水','广安','亳州','昌吉','鄂州','白城','恩施','伊犁','铜陵','林芝','海东','桂林','临汾','渭南','滨州','菏泽','孝感','营口','濮阳','抚顺','自贡','新余','鸡西','钦州','淮北','安康','资阳','乌海','中卫','鹤岗','石嘴山','日喀则','昌都','德宏州','吐鲁番','阿勒泰','临沂','威海','泰安','宿迁','德州','安阳','驻马店','绥化','上饶','莆田','蚌埠','大庆','漯河','十堰','晋城','宣城','舟山','邵阳','大理','楚雄','天水','池州','松原','阜新','巴音郭楞','仙桃','武威','普洱','河池','铜川','迪庆州','潜江','贺州','塔城','济宁','江门','赣州','许昌','滁州','龙岩','运城','长治','乐山','郴州','遂宁','曲靖','呼伦贝尔','阳江','朝阳','巴彦淖尔','朔州','娄底','梧州','巴中','黔南','吴忠','临夏','凉山州','定西','萍乡','甘南','攀枝花','克孜勒苏','山南','临沧','金昌','邢台','衡水','宁德','榆林','鞍山','焦作','牡丹江','齐齐哈尔','眉山','三明','吕梁','吉安','平凉','通辽','揭阳','黄石','宜宾','延边','汕尾','玉林','通化','四平','阿克苏','贵港','白山','双鸭山','湘西','哈密','保山','鹰潭','喀什','雅安','黑河','和田','固原','阿拉善盟','七台河','嘉峪关'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 8c12e22b0..03d4cfcb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -67,7 +67,7 @@ class ThirdPartyPriceBatch extends BaseBatch { } def header = []//["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] - FileUtils.batchCreateFile(rows, fileName, header, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName, header, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index c5b66e238..770ea6914 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -32,8 +32,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def sqlInstance = null public static def batchsize = 1000 static main(args) { @@ -43,6 +42,13 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { def citys = CcbCommonUtils.getRunningCitys(args) + if(citys.size()==92){ + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + }else{ + sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + } def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index f033823f9..ce6242a31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -183,7 +183,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, region) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 0ca1bc62a..8e74425cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -251,7 +251,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern(startMonth + "-20", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern(startMonth + "-30", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e, regionText) From 55368a0a9fe0541d0c7410f69e1d0527dbfd2761 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Nov 2022 17:55:15 +0800 Subject: [PATCH 2/2] update --- .../ccbevaluation/BigdataDealBatch.groovy | 59 ++++++++++++------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 86b64e4db..2803b3229 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -63,7 +63,7 @@ class BigdataDealBatch extends BaseBatch{ curDate = args[1].toString().toInteger() } - println "curDate:"+curDate +// println "curDate:"+curDate // for(def city : citys){ //// if(!ccbCitys.contains(city)){ // addDealPriceMatch(city); @@ -87,16 +87,16 @@ class BigdataDealBatch extends BaseBatch{ } - def trades = getCityData(city, statMap, ppeDists) - if(trades){ - def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") - rows.addAll(rets) - println "【${index}】"+city +" 成交共:"+rets.size() - - def cityDatas = [] - cityDatas << headers - cityDatas.addAll(rows) - } +// def trades = getCityData(city, statMap, ppeDists) +// if(trades){ +// def rets = createData(trades, cityCodeDataMap, distCodeMap, "other") +// rows.addAll(rets) +// println "【${index}】"+city +" 成交共:"+rets.size() +// +// def cityDatas = [] +// cityDatas << headers +// cityDatas.addAll(rows) +// } } @@ -105,7 +105,7 @@ class BigdataDealBatch extends BaseBatch{ fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } - FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", true); + FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); println "txt 生成 " + rows.size() } @@ -114,9 +114,11 @@ class BigdataDealBatch extends BaseBatch{ def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) - def lianjiaPriceMap = findThirdPartyPrice("上海") - def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') +// def lianjiaPriceMap = findThirdPartyPrice("上海") +// def startDate = DateUtils.parseFromTightByPattern(DateUtils.addMonth(new Date(),-2),'yyyy-MM-dd') // def lianjiaPriceMap = getDistrictStatByCity("上海", startDate) + + def ccbPriceMap = getCommunityPrices("上海") for(def i=0;i