From 330d1fcb5f977f26f192ecadc74c123f168c917a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sat, 7 Oct 2023 16:43:46 +0800 Subject: [PATCH] update --- .../groovy/com/fangjia/script/ana/Test.groovy | 43 ++++- .../script/ccbevaluation/AddSzDealData.groovy | 4 +- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 4 +- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 79 ++++----- .../script/commData/CommonDatautils.groovy | 6 +- .../fuzhou/DistrictInfoExportBatch.groovy | 5 +- .../fuzhou/DistrictPriceExportBatch.groovy | 1 + .../fuzhou/DistrictSellStatBatch.groovy | 15 +- .../script/fuzhou/FzDataResultStat.groovy | 36 +++- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../fangjia/script/gj/data/FieldStat.groovy | 14 +- .../script/gj/district/AddAlias.groovy | 44 +++-- .../gj/district/DistrictFieldAddBatch.groovy | 90 ++++++++-- .../gj/district/FangjiaClientUtils.groovy | 18 ++ .../script/gj/district/PricePkTest.groovy | 42 +++++ .../gj/districtwash/DistrictTypeUpdate.groovy | 2 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 44 ++--- .../script/gj/township/StreetProcess.groovy | 13 +- .../fangjia/script/gj/xiamen/PoiWash.groovy | 158 ++++++++++++++++++ .../script/gj/xiamen/SeaweedDataSynch.groovy | 19 ++- .../fangjia/script/wenyan/temp/ZhuoXin.groovy | 8 +- 25 files changed, 529 insertions(+), 128 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/PricePkTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/Test.groovy index 71dbe7a28..95859320b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/Test.groovy @@ -1,5 +1,7 @@ package com.fangjia.script.ana +import com.fangjia.script.util.ExcelUtils + import java.util.Date; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -40,9 +42,46 @@ public class Test { // //test.run("district", "上海"); // test.analyDistrictPk("district", "天津");//小区统计与小区的对比 // println new Date(new ObjectId('5469bb4cda29ee73010fa577').getTime()) - test.addTestData(); +// test.addTestData(); + test.findRepeatGuidingPrice() + } + + def findRepeatGuidingPrice(){ + def q = new BasicDBObject(); + q.put("guiding_date", new BasicDBObject('$gte', DateUtils.parseFromTightByPattern("2023-03-01", "yyyy-MM-dd"))) + def dbcursor = GUIDING_PRICE_COLL.find(q) + def map = [:] + for(def d : dbcursor){ + def key = d.city +"##"+d.region+"##"+d.name + def rets = map.get(key) + if(!rets){ + rets = [] + map.put(key, rets) + } + rets << d + } + + def datas = [] + datas << ["id", "城市","区域","名称","添加人","指导价","日期"] + map.each{key,list-> + if(list.size()>1){ + def userNames = [] + for(def ret : list){ + userNames << ret.guiding_person + } + if(userNames.contains("付贵生")){ + for(def r : list){ + if(r.guiding_person!="付贵生"){ + GUIDING_PRICE_COLL.remove(new BasicDBObject("_id", r._id)) + } + } + + } + } + } + +// ExcelUtils.write("d:\\数据\\福州\\指导价.xlsx",datas, "datas") } - public void addTestData(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index 1453028a4..6f83a19e0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -9,7 +9,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳7月份二手房成交数据汇总.xls", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\深圳8月份二手房成交数据汇总.xls", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) @@ -74,7 +74,7 @@ for(def d : datas){ if(trade_date){ map.trade_date = trade_date }else{ - def trade_date1 = DateUtils.getRandomDate("2023-05-16", "2023-05-31") + def trade_date1 = DateUtils.getRandomDate("2023-08-01", "2023-08-31") map.trade_date = DateUtils.parseFromTightByPattern(trade_date1, "yyyy-MM-dd") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 03bc9ebbe..4139bce46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -39,7 +39,7 @@ class BigdataDealBatch extends BaseBatch{ def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) - def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] + def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com", "幸福里":"xflapp.com"] public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index c8058a931..7546a13d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -37,7 +37,7 @@ class CcbCommonUtils { public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) - public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42"] + public static def rootMap = ["lianjia.com":"1","anjuke.com":"2","fang.com":"3","ke.com":"4","58.com":"5","leju.com":"6","fccs.com":"7","c21.com.cn":"8","qfang.com":"9","ganji.com":"10","leyoujia.com":"11","fangdd.com":"12","5i5j.com":"13","centanet.com":"14","zhuge.com":"15","baixing.com":"16","house365.com":"17","izhiliao.com":"18","wkzf.com":"19","jiwu.com":"20","xyfcw.com":"21","917.com":"22","0597house.com":"23","xmfish.com":"24","rongyizhaofang.com":"25","rz0375.com":"26","596fc.com":"27","zhen22.com":"28","taotaoju.vip":"29","kaifc.cn":"30","zhongmou.cc":"31","0391fc.com":"32","hualongxiang.com":"33","fangtan007.com":"34","h0591.com":"35","0598yu.com":"36","zfgou.cn":"37","0597ok.com":"38","second.cc":"39","rent.cc":"40","ssfcw.net":"41","czfcw.com":"42","xflapp.com":"43"] public static def specialCityMap = ["阿坝州":"阿坝","阿拉善盟":"阿拉善","黄南州":"黄南","巴音郭楞":"巴州","兴安盟":"兴安","锡林郭勒盟":"锡林郭勒","凉山州":"凉山","西双版纳州":"西双版纳","德宏州":"德宏","迪庆州":"迪庆","海北州":"海北","海西州":"海西","博尔塔拉":"博州","甘孜州":"甘孜","玉树州":"玉树","果洛州":"果洛","文山州":"文山","怒江州":"怒江","克孜勒苏":"克州"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index c2b5ce129..641263d9e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -392,12 +392,12 @@ class DealPriceDataProcess { def deleteThirdPartyData(){ def citys = CcbCommonUtils.citys_all - def month = 202307 + def month = 202308 def rows = [] def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def set = new HashSet() for(def city : citys) { - def rets = findPeGteThirdParty2(city, 202308) + def rets = findPeGteThirdParty2(city, 202309) def total = 0 println city +" "+rets.size() if(rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 67ea43dda..ff83535fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -52,7 +52,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2023-07-26", "2023-08-25") + def prices = CcbCommonUtils.findPersonPrice(city, "2023-08-25", "2023-09-27") // def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-01", "2022-11-22") if(prices){ def distMap = getDistrictMap(city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0fd8eaa49..3f069821a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.RandomUtil import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.CcbCodeUtils @@ -1287,10 +1288,10 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\录入国信达价格_0602.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\9.25人工指导价.xlsx" -//ExcelUtils.multiThreadingModify(fileName, {d-> -// if(d['A']){ +ExcelUtils.multiThreadingModify(fileName, {d-> + if(d['A']){ // def region = RegionData.formatRegion(d['B'], d['C']) // if(region){ // def dist = CommonDatautils.findDistrict(d['B'], region, d['E'], "district") @@ -1299,18 +1300,18 @@ def fileName = "D:\\ // d['H'] = dist._id.toString() // } // } -//// if(!d['F']){ -//// def dist = CommonDatautils.findDistrictById(d['A']) -////// -//// if(dist){ -////// d['H'] = dist.region -////// d['G'] = dist.name -//// d['F'] = dist.status -//// } -//// } -// -// } -//},0) + if(!d['K']){ + def dist = CommonDatautils.findDistrictById(d['E']) + if(dist){ + d['K'] = dist.region +// d['I'] = dist.name +// d['J'] = dist.status +// def val = d['F'].toString().toDouble().toInteger() + RandomUtil.random(-100,500) +// d['K'] = val.toInteger() + } + } + } +},0) @@ -1373,30 +1374,30 @@ def findPgDealPrice(c, pgsqlInstance, startDate){ return count } -citys = CcbCommonUtils.citys_all -def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() -//def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"] -def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] -def rows = [] -def headers = [] -headers << "城市" -headers.addAll(dates) -rows << headers -for(def c : citys){ - def row = [] - row << c - def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c)) -// for(def d : dates){ - def startDate = "2022-07-01" - def count = findPgDealPrice(c, pgsqlInstance, startDate) - row << count*100/disCount - -// def map = findSampleRate(pgsqlInstance, c, d) -// row << map.third -// } - rows << row -} -ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方") +//citys = CcbCommonUtils.citys_all +//def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +////def dates = ["2022-07-01","2022-08-01","2022-09-01","2022-10-01","2022-11-01","2022-12-01","2023-01-01","2023-02-01","2023-03-01","2023-04-01","2023-05-01","2023-06-01"] +//def dates = [202208,202209,202210,202211,202212,202301,202302,202303,202304,202305,202306] +//def rows = [] +//def headers = [] +//headers << "城市" +//headers.addAll(dates) +//rows << headers +//for(def c : citys){ +// def row = [] +// row << c +// def disCount = PgsqlCommUtils.findAllDistrictCount(cityCodeDataMap.get(c)) +//// for(def d : dates){ +// def startDate = "2022-07-01" +// def count = findPgDealPrice(c, pgsqlInstance, startDate) +// row << count*100/disCount +// +//// def map = findSampleRate(pgsqlInstance, c, d) +//// row << map.third +//// } +// rows << row +//} +//ExcelUtils.write("D:\\数据\\建行\\数据统计结果\\成交占比统计-年.xlsx", rows, "第三方") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index eee6db626..5d4fe0bbd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -302,7 +302,11 @@ class CommonDatautils { q.put("cat", cat); q.put("city", city); if(startDate){ - q.put("m_date", new BasicDBObject('$gte', startDate).append('$lt',endDate)) + def datequery = new BasicDBObject('$gte', startDate) + if(endDate){ + datequery.append('$lt',endDate) + } + q.put("m_date", datequery) } DBObject keys = new BasicDBObject(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy index 206e8c11a..377a7a6c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictInfoExportBatch.groovy @@ -97,7 +97,7 @@ class DistrictInfoExportBatch extends BaseBatch{ saveOrUpdateFzDist(districts) if(day == 15){ - def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), endMDate); + def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, DateUtils.parseFromTightYYYYMMDD("2022-01-01"), null); remarkRemoveDist(removeDists) }else{ def removeDists = CommonDatautils.findRemoveDistricts(city, "district", null, startMDate, endMDate); @@ -201,6 +201,9 @@ class DistrictInfoExportBatch extends BaseBatch{ def udate = new BasicDBObject() udate.put("status",1) udate.put("m_date", rd.m_date) + udate.put("region", rd.region) + udate.put("name", rd.name) + udate.put("estate_type2", rd.estate_type2) FZ_DIST_COLL.update(new BasicDBObject('_id',rd.get('_id')), new BasicDBObject('$set', udate)); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 4ec710a3c..d18bc364e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -131,6 +131,7 @@ class DistrictPriceExportBatch extends BaseBatch{ for(def rMap : districts){ def curPrice = 0 def dbo = null + if(curPriceMap.containsKey(rMap.get("_id").toString())){ dbo = curPriceMap.get(rMap.get("_id").toString()) curPrice = dbo.avg_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy index 4e3d5668e..d88fa4037 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictSellStatBatch.groovy @@ -99,8 +99,10 @@ class DistrictSellStatBatch extends BaseBatch { println index+" "+city +" "+districts.size()+" "+ (new Date()).toString() + def startDate = DateUtils.getFirstDayOfMonth(new Date(), -1); + def endDate = DateUtils.getLastDayOfMonth(new Date(), -1); if(districts.size()>0){ - def weekendPriceMap = findPriceMap(city, lastMonthWeekend)//上个月小区均价 + def weekendPriceMap = findPriceMap(city, startDate, endDate)//上个月小区均价 def curMonthlyPriceMap = findMonthlyPrice(city, curMonth)//当月小区月均价 def lastMonthlyPriceMap = findMonthlyPrice(city, monDate)//上个月小区月均价 def lastYearPriceMap = findMonthlyPrice(city, yoyDate)//去年小区月均价 @@ -245,11 +247,12 @@ class DistrictSellStatBatch extends BaseBatch { } - def findPriceMap(city, weekend){ + def findPriceMap(city, startDate, endDate){ def q = new BasicDBObject() q.put("city", city) - q.put("weekend", weekend) - def dbcursor = DISTRICT_STAT.find(q) +// q.put("district_name","金福花园") + q.put("weekend", new BasicDBObject('$lte', endDate).append('$gte', startDate)) + def dbcursor = DISTRICT_STAT.find(q).sort(new BasicDBObject().append("weekend", -1)) def priceMap = [:] for(def d : dbcursor){ if(city == "福州"){ @@ -257,7 +260,9 @@ class DistrictSellStatBatch extends BaseBatch { d.avg_price = d.avg_price_2b } } - priceMap.put(d.region +"##"+d.district_name, d) + if(!priceMap.containsKey(d.region +"##"+d.district_name)){ + priceMap.put(d.region +"##"+d.district_name, d) + } } return priceMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index b2d9b9338..02d267a37 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -22,6 +22,10 @@ class FzDataResultStat { public static final DBCollection FZ_DISTRICT_STAT_COLL = MongoUtils.getCollectionByName("business", "fz_district_stat", "mongo.db.fangjia.com") + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection FZ_DIST_COLL = MongoUtils.getCollectionByName2("business", "fz_district", lineUrl) + + static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); @@ -55,7 +59,8 @@ class FzDataResultStat { def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); // List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) - def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"]) +// def dists = CommonDatautils.findAllDistricts("福州", "district", ["status"]) + def dists = getFzDistricts("福州", ["status"]) def idMaps = [:] for(def d : dists){ idMaps.put(d._id.toString(), d.status==0?"有效":"无效") @@ -82,7 +87,7 @@ class FzDataResultStat { map2.put(key, obj) } def rows = [] - rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅", "是否有效"] + rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅","计算方式", "是否有效", "涨跌绝对值"] def index = 0 map1.each {k,obj1-> def keys = k.split("##") @@ -102,7 +107,12 @@ class FzDataResultStat { if(obj1){ flag = idMaps.get(obj1.comm_id) } - def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag] + + def markup_abs = null + if(markup){ + markup_abs = Math.abs(markup) + } + def row = [obj1.city, obj1.region, obj1.name, obj1?obj1.avg_price:'', lastPrice?lastPrice.avg_price:'', markup,obj1?obj1.value_type:'', flag, markup_abs] rows << row index++ println row @@ -263,4 +273,24 @@ class FzDataResultStat { q.put("month", lastMonth) return FZ_DISTRICT_STAT_COLL.find(q).toList() } + + def getFzDistricts(def city, fields){ + def q = new BasicDBObject() + q.put("city", city) + q.put("cat", "district") + def rets = [] + + DBObject keys = new BasicDBObject(); + if(fields){ + for(def field : fields){ + keys.put(field, 1); + } + } + + + FZ_DIST_COLL.find(q, keys).each{it-> + rets << it + } + return rets + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 8e3147530..c971794fd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202307.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202308.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index f06dd1590..488870b22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,11 +48,11 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2023-07-01" + static def statDate = "2023-08-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo -// task.esfParser("D:\\数据\\友达\\2023年8月\\查询1.DBF");//友达二手房存入mongo + task.esfParser("D:\\数据\\友达\\2023年9月\\查询1.DBF");//友达二手房存入mongo // task.newhouseToSaveEs();//新房保存到es task.esfhouseToSaveEs();//二手房保存到es // task.updateFinishDateByEsfTrade();//根据二手成交的竣工时间修改seaweed库中的竣工时间 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy index 62666390b..a342f7438 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/FieldStat.groovy @@ -14,12 +14,12 @@ class FieldStat { static main(args) { def task = new FieldStat(); // task.run() -// task.run1() + task.run1() // task.run2() // task.run3() // task.offircRun() - task.distRun() - task.shopRun() +// task.distRun() +// task.shopRun() } def distRun(){ @@ -203,11 +203,15 @@ class FieldStat { alias_total = alias_total + alias.size() } } - def precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size()) + def precent = 0 + if(districts){ + precent = DataHandleUtils.getDoubleRound(2, alias_index*100.0/districts.size()) + } + rets << [city, alias_total, alias_index, districts.size(), precent+"%"] } - ExcelUtils.write("d:\\数据\\92个城市别名统计.xlsx", rets, "统计") + ExcelUtils.write("d:\\数据\\城市别名统计.xlsx", rets, "统计") // println dist_total // println alias_total // println addr_alias_total diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy index 7b5b4df94..5d4b81290 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/AddAlias.groovy @@ -5,24 +5,38 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject +// +//ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d-> +// def opt = d['J'] +// if(opt && opt.contains("别名")){ +// def dist = CommonDatautils.findDistrictById(d['H']) +// if(dist){ +// def alias = [] +// if(dist.alias){ +// alias = dist.alias +// } +// if(!alias.contains(d['C'])){ +// alias << d['C'] +// def udata = new BasicDBObject() +// udata.put("alias", alias) +// DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) +// } +// +// } +// } +//}) -ExcelUtils.modify("D:\\数据\\常州\\有样本小区-匹配 -区域相同样本量超过等于3条(人工手动).xlsx",{d-> + + +ExcelUtils.modify("D:\\数据\\厦门\\厦门户数(曹和付)-结果.xlsx",{d-> def opt = d['J'] - if(opt && opt.contains("别名")){ - def dist = CommonDatautils.findDistrictById(d['H']) + if(opt && opt.toString().toDouble().toInteger()==1){ + def dist = CommonDatautils.findDistrict('厦门',d['G'],d['H'], "district") if(dist){ - def alias = [] - if(dist.alias){ - alias = dist.alias - } - if(!alias.contains(d['C'])){ - alias << d['C'] - def udata = new BasicDBObject() - udata.put("alias", alias) - DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) - } - + def udata = new BasicDBObject() + udata.put("household_count", d['F'].toString().toDouble().toInteger()) + DbUtils.seaweed.update(new BasicDBObject('_id', dist.get("_id")), new BasicDBObject('$set', udata)) } } - }) + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 6aac3ebde..400e81699 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -39,17 +39,22 @@ class DistrictFieldAddBatch extends BaseBatch{ //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" // - def sourceMap = ["安居客-新房":"安居客-新房","禧泰":"中国房价行情","安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", - "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源 + def sourceMap = ["安居客":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", + "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源 + + //"alias":"alias", //ownership_type:权属类型 为商品房、经适房、动迁房等 //house_property 产权性质:国有产权 "house_property":"网站没有", - def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", - "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", - "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", - "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", - "loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段 +// def fieldsMap = ["estate_charge":"property_fee","block":"block","build_date":"build_year","estate_company":"property_company","developer":"developer", +// "virescence_percent":"greening_rate","cubage_percent":"plot_ratio","build_area":"build_area","all_area":"land_area","household_count":"households", +// "car_site_count":"parking_cnt","car_site":"parking_cnt","property_life":"land_usage_period","address":"address","build_structure":"building_structure", +// "build_type":"build_type", "land_property":"ownership_type","live_date":"live_date", +// "loop_line_info":"loop_line","loudong_count":"loudong_cnt"]//key:seaweed表字段 value:es表字段 + + + def fieldsMap = ["household_count":"households"] def fieldsNameMap = ["estate_charge":"物业费","block":"板块","build_date":"建筑年代","estate_company":"物业公司","developer":"开发商", "virescence_percent":"绿化率","cubage_percent":"容积率","build_area":"建筑面积","all_area":"占地面积","household_count":"户型", @@ -88,8 +93,8 @@ class DistrictFieldAddBatch extends BaseBatch{ for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) if(totalCount){ -// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets) - analysRets = addFieldFromMatch(c, totalCount, analysRets) +// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合 + analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配 } } println analysRets.toString() @@ -106,11 +111,11 @@ class DistrictFieldAddBatch extends BaseBatch{ def matchRets2 = selectRestByCredit(matchRets1) println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size() - def updateRets = processFieldsFromMatch(c , matchRets2) + def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource) println "【通过匹配填充字段】网站:"+seaweedmapSource +" Es:"+esRets.size()+" 融合小区占比:"+DataHandleUtils.getDoubleRound(2, matchRets2.size()*100/totalCount)+"%" def precentMap = fieldUpdate(totalCount, updateRets) println c +" "+precentMap.toString() - updateDB(c, updateRets) + //updateDB(c, updateRets) println "------------------------------------------------------------------------------" def pre1 = 0 def pre2 = 0 @@ -199,11 +204,13 @@ class DistrictFieldAddBatch extends BaseBatch{ } - def processFieldsFromMatch(c,matchRets){ + def processFieldsFromMatch(c,matchRets, esWebSource){ def fields = fieldsMap.keySet() as List fields << "city" fields << "estate_type2" def fieldValueMap = [:] + def rets = [] + rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"] for(def matchDist : matchRets){ def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields) if(!dist){ @@ -212,11 +219,13 @@ class DistrictFieldAddBatch extends BaseBatch{ } def newValues = strandField(c, dist, matchDist) if(newValues){ + rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag] println dist._id.toString()+" "+newValues.toString() fieldValueMap.put(dist._id.toString(), newValues) } } + ExcelUtils.write("D:\\数据\\小区\\${c}户数.xlsx",rets,esWebSource) return fieldValueMap; } @@ -332,7 +341,7 @@ class DistrictFieldAddBatch extends BaseBatch{ def findEsDatas(c, esWebSource){ def list = [] - ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}'", {r-> + ElasticUtils.executeSqlQuery("select * from community where city='${c}' and source='${esWebSource}' ", {r-> list << r }) return list @@ -379,6 +388,7 @@ class DistrictFieldAddBatch extends BaseBatch{ try{ totalCount.addAndGet(1) def ret = matching(data) +// def ret = newMatching(data) if(ret){ // println totalCount.get() +" "+data.city+" "+data.name+" [Match] "+data.match_region+" "+data.match_name +" "+data.flag+" "+data.credit rets << ret @@ -399,7 +409,7 @@ class DistrictFieldAddBatch extends BaseBatch{ def selectRestByCredit(matchRets){ def matchRets1 = [] for(def r : matchRets){ - if(r.credit && r.credit>=0.9){ + if(r.credit && r.credit>0.9){ matchRets1 << r } } @@ -458,6 +468,58 @@ class DistrictFieldAddBatch extends BaseBatch{ } + def newMatching(data){ + def name = data.name + if(data.name && data.name.contains("·")){ + name = data.name.replace("·", "") + } + + def flag = "名称匹配" + def dist = matchDist(data.city, data.region, name) + + if(!dist){ + dist = CommonDatautils.findSeaweedByMappingAlias(data.city, data.region, name,"district") + flag = "映射别名匹配" + } + if(!dist) { + def rets = CommonDatautils.findDistrictByAlias(data.city, data.region, name, "district") + if(rets && rets.size()==1){ + flag = "别名匹配" + dist = rets[0] + } + } + + if(dist){ + data.flag = flag + data.match_id = dist._id.toString() + data.match_region = dist.region + data.match_name = dist.name + data.credit = 0.95 + }else{ + def jsonObject = FangjiaClientUtils.newMatch(data.city, data.region+name, "property", false) + if(jsonObject!=null && jsonObject.credit>0){ + if(data.address && isStandardRoad(data.address)){ + jsonObject = FangjiaClientUtils.newMatch(data.city, data.address, "property", false) + }else{ + data.flag = "匹配接口-地址" + } + }else{ + data.flag = "匹配接口-名称" + } + if(jsonObject != null && jsonObject.credit>0) { + data.match_id = jsonObject.get("id") + data.match_region = jsonObject.get("district") + data.match_name = jsonObject.get("name") + data.credit = jsonObject.get("credit") + } + } + if(data.match_id){ + return data + } + return null + } + + def updateDB(c, updateRets){ for(def rm : updateRets) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 210fa41c8..9da0ae032 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -200,4 +200,22 @@ class FangjiaClientUtils { return jsonObj; } + + public static def newMatch(city,address, property, useMap){ + def jsonObj = null + def content = null + try { + address = address.replace("{","") + // def url = "http://192.168.0.19:39080/FangJiaNlpService/ner?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&city="+URLEncoder.encode(city,"UTF-8")+"&category=${property}&useMap=${useMap}" +// def url = "http://192.168.0.19:39080/FangJiaAddressService/structure?address="+URLEncoder.encode(address,"UTF-8") + content = new HttpRequest().newGet(url, null, "utf-8") + jsonObj = JSONObject.fromObject(content) + } catch (Exception e) { + println address +" "+content + e.printStackTrace(); + } + return jsonObj; + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/PricePkTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/PricePkTest.groovy new file mode 100644 index 000000000..975d618ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/PricePkTest.groovy @@ -0,0 +1,42 @@ +package com.fangjia.script.gj.district + +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; +DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + +def city = "常州" +def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "alias"]) +println districts.size() +def rets = [] +rets << ["城市", "区域", "名称", "房价网ToC", "房价网ToB", "近三个月成交均价"] +def index = 0 +for(def d : districts){ + index++ + def ret = [] + def dbo = CommonDatautils.queryDistrictPrice(d.city, d.region, d.name, DateUtils.parseFromTightByPattern("2023-09-17","yyyy-MM-dd")) + ret << d.city + ret << d.region + ret << d.name + if(dbo){ + ret << dbo.avg_price + ret << dbo.avg_price_2b + } + + + def map = CcbCommonUtils.findTrades(d, "2023-05-01","2023-09-01") + if(map){ + ret << map.avg_price + } + rets << ret + println index +} +ExcelUtils.write("d:\\数据\\江南农商行\\常州小区价格.xlsx", rets, "数据") + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictTypeUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictTypeUpdate.groovy index f8e1b8424..8608b9a8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictTypeUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictTypeUpdate.groovy @@ -15,7 +15,7 @@ class DistrictTypeUpdate { } def updateType(){ - ExcelUtils.modify("D:\\数据\\厦门\\厦门银行涉及城市(物业类型可疑小区).xlsx",{d-> + ExcelUtils.modify("D:\\数据\\厦门\\物业类型可疑小区(第二批).xlsx",{d-> def flag = d['F'] if(flag && flag.toString().toDouble().toInteger()==2){ def name = d['C'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index bcfef9581..22a51985b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -43,6 +43,7 @@ public class LianjiaCrawlByChrome { // task.addDealRegion(); // task.standardRegion() // task.updadteDealPriceField() + } def updadteDealPriceField(){ @@ -263,26 +264,27 @@ public class LianjiaCrawlByChrome { } def checkDocument(driver, url){ - driver.get(url) - def s = driver.getPageSource() - def flag = true - while(flag){ - try{ - if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ - driver = getLogin(driver, url) - driver.get(url) - Thread.sleep(5000) - s = driver.getPageSource() - }else{ - flag = false - } - }catch(Exception e){ - driver.close() - driver = new ChromeDriver() - Thread.sleep(15000) - driver = getLogin(driver, url) - println "异常Url:"+url + try{ + driver.get(url) + def s = driver.getPageSource() + def flag = true + while(flag){ + + if(s && (s.contains("无法访问此网站") || s.contains("登录"))){ + driver = getLogin(driver, url) + driver.get(url) + Thread.sleep(5000) + s = driver.getPageSource() + }else{ + flag = false + } } + }catch(Exception e){ + driver.close() + driver = new ChromeDriver() + Thread.sleep(15000) + driver = getLogin(driver, url) + println "异常Url:"+url } return driver; } @@ -581,11 +583,11 @@ public class LianjiaCrawlByChrome { def cityDoc = Jsoup.parse(driver.getPageSource()); def cityEles = JQueryUtils.select(".city_province a", cityDoc) def cityList = [] - def citys= ["衢州"] + def citys= ["福州"] if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && citys.contains(e.text())){ + if (!href.contains("fang.lianjia")){ cityList << [e.text(), href] } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy index abb5bedc4..967fa21ce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/township/StreetProcess.groovy @@ -31,23 +31,24 @@ class StreetProcess { task.run() } def run(){ - - def region = RegionData.formatRegion("三明市", "三元区") - println region +// +// def region = RegionData.formatRegion("三明市", "三元区") +// println region // standard() // matchStreet() -// synchTownship() + synchTownship() } def synchTownship(){ def q = new BasicDBObject() // q.put("city_name","福州市") - q.put("shape",new BasicDBObject('$exists', true)) +// q.put("shape",new BasicDBObject('$exists', true)) def dbcursor = TOWNSHIP.find(q) for(def d : dbcursor){ def udata = new BasicDBObject() - udata.put("shape", d.shape) +// udata.put("shape", d.shape) + udata.put("area_station", d.area_station) TOWNSHIP100.update(new BasicDBObject('_id', d.get("_id")), new BasicDBObject('$set', udata)) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy new file mode 100644 index 000000000..b4266d748 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/PoiWash.groovy @@ -0,0 +1,158 @@ +package com.fangjia.script.gj.xiamen + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.rabbitmq.client.impl.AMQImpl + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class PoiWash { + + def MAP_POI_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection MAPPOI_CONN = MongoUtils.getCollectionByName2("fangjia", "map_poi_test", MAP_POI_URL) + DBCollection TOWNSHIP_CONN = MongoUtils.getCollectionByName2("fangjia", "township", MAP_POI_URL) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + + public static def townsMap = [:] + + static main(args) { + def task = new PoiWash() + task.run() + } + + def run(){ + + //"三明市", + def citys = ["宁德市","泉州市","厦门市","福州市","漳州市","南平市","龙岩市","莆田市","重庆市"] + for(def city : citys){ + townsMap = findAllTownship(city) + def datas = findPois(city) + println city +' '+datas.size() + process(city, datas) + println city +" 完成" + } + } + + def process(city, results){ +// def rets = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + def updateTotalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(results.size()) + + def count = results.size() + for(def i=0;i OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20230801", "yyyyMMdd") -def endDate = DateUtil.parseDate("20230901", "yyyyMMdd") +def startDate = DateUtil.parseDate("20230901", "yyyyMMdd") +def endDate = DateUtil.parseDate("20231001", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 @@ -23,7 +23,7 @@ while (startDate.before(endDate)) { def nextDate = calendar.getTime() count++ ///property/addressEvaluate /address/financialRating /v2/property/detail - OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/address/financialRating"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + OPEN_URI_LOG.find(and(eq("user_id", "1007779"), eq("request_uri", "/v2/property/detail"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> def result = JSONObject.fromObject(doc.res_map.result) // if (!hashSet.contains(result)) { def item = [doc._id.toString(), "辽宁卓信普惠征信", doc.ip, doc.request_uri, doc.c_date.format("yyyy-MM-dd HH:mm:ss"), doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] @@ -39,6 +39,6 @@ while (startDate.before(endDate)) { startDate = nextDate } -ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202309.xlsx", data, "财力评级") +ExcelUtils.write("D:\\数据\\卓信\\辽宁卓信普惠征信-202310.xlsx", data, "楼盘评情")