From d23244e83929d6caa8c31b7903d1e895cd9aedee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Aug 2024 18:23:36 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=AF=B9=E7=A7=81=E4=B8=8D=E7=94=A8?= =?UTF-8?q?=E9=BD=90=E6=A2=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/AddrSvcController.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java index aa1183398..392a896a4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddrSvcController.java @@ -441,6 +441,8 @@ public class AddrSvcController { if (result.containsKey("devDistrictName")) { basicModel.setResult(result); + } else if (!"2".equals(type)) { // 对私的不用齐梁 + basicModel.setResult(result); } else { try { //如果是江苏省或者浙江省 调用齐梁的结果 @@ -538,7 +540,8 @@ public class AddrSvcController { //我们省级及以上有开发区,直接用我们的结果 if (result.containsKey("devDistrictName")) { basicModel.setResult(result); - + } else if (!"2".equals(type)) { // 对私的不用齐梁 + basicModel.setResult(result); } else { try { //如果是江苏省或者浙江省 调用齐梁的结果 @@ -674,7 +677,8 @@ public class AddrSvcController { //我们省级及以上有开发区,直接用我们的结果 if (result.containsKey("devDistrictName")) { basicModel.setResult(result); - + } else if (!"2".equals(type)) { // 对私的不用齐梁 + basicModel.setResult(result); } else { try { //如果是江苏省或者浙江省 调用齐梁的结果 From e52540856f03f7dcae5137a0cf81bc8ec886a7f0 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 14 Aug 2024 14:14:25 +0800 Subject: [PATCH 2/5] update --- .../script/commData/CommonDatautils.groovy | 6 ++ .../script/gj/data_handle/DevProcess.groovy | 39 ++++++----- .../data_handle/ThardParthAnalyBatch.groovy | 68 +++++++++++++++++-- 3 files changed, 91 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index f197f974c..d36b853dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1209,6 +1209,12 @@ class CommonDatautils { return rets; } + public static def findAllCount(String cat){ + DBObject q = new BasicDBObject(); + q.put("cat", cat); + return SEAWEED_MONGO_COLL.count(q) + } + public static def findDistrictsByName(String city, String name, String cat){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index f90ed42f6..772bbaf62 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -37,7 +37,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -69,7 +69,7 @@ class DevProcess { // task.addKfc() // task.devToPark() //开发区下辖园区 - task.checkTown() +// task.checkTown() } @@ -349,22 +349,27 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\新疆开发区名单.xlsx",{d-> - if(!d['Q'] && d['A']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> + if(!d['I'] && d['J']){ def q = new BasicDBObject() -// q.put("name", d['A']) +// q.put("name", d['J']) // q.put("city",d['G']) -// q.put('alias', new BasicDBObject('$in',[d['A']])) - q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+d['A']+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 +// q.put('alias', new BasicDBObject('$in',[d['J']])) + def name = d['J'].replace('经济技术开发区','经济开发区') + q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 def obj = development_zone_CONN.findOne(q) if(obj){ - d['C'] = obj._id.toString() - d['D'] = obj.name - if(obj.name == d['A']){ - d['E'] = "相同" - }else{ - d['E'] = "包含" - } + d['I'] = "库中有开发区-相似2" + d['K'] = obj.name +// d['C'] = obj._id.toString() +// d['D'] = obj.name +// if(obj.name == d['A']){ +// d['E'] = "相同" +// }else{ +// d['E'] = "包含" +// } +// d['E'] = "别名" +// d['F'] = obj.name } } },0); @@ -913,12 +918,12 @@ class DevProcess { def checkTown(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx",{d-> + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> if(!d['E']){ - if(d['C'] && d['D']){ + if(d['A']){ // if(d['A']=="长春高新技术产业开发区"){ - def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B']) + def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A']) if(jsonObject1) { if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { def township = jsonObject1.get("township") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy index ef6476d48..ee60d55b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -1,14 +1,19 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.datastore.driver.FacetResult import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.AggregationOutput import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject @BatchOwner(Watcher.GaoJuan) class ThardParthAnalyBatch extends BaseBatch{ @@ -31,22 +36,36 @@ class ThardParthAnalyBatch extends BaseBatch{ } def run(){ + def fjwTotalCount = CommonDatautils.findAllCount("district") def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd") def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); println "统计月份:"+sDate+" 查询更新时间:"+mdate def crawlTotalCount = findAllEsDataCount(mdate) - println "抓取第三方小区量:"+crawlTotalCount + println "抓取第三方小区总量:"+crawlTotalCount def synchTotalCount = findAllMongoDatasCount(sDate) - println "同步到第三方库小区量: "+synchTotalCount + println "同步到第三方库小区总量: "+synchTotalCount + def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate) println "同步到第三方库有价格数量:"+synchHavePriceTotalCount + + def synchDistCount = findAllMongoDistrictCount(sDate) + println "同步到第三方库有价格小区量: "+synchDistCount + def matchTotalCount = findMatchSuccessCount(sDate) - println "第三方小区匹配成功量:"+matchTotalCount + println "第三方匹配成功总量:"+matchTotalCount + + def matchDistTotalCount = findMatchSuccessDistCount(sDate) + println "第三方匹配成功小区量:"+matchDistTotalCount + + def precent = matchDistTotalCount*100.0/fjwTotalCount + precent = DataHandleUtils.getDoubleRound(2, precent) + println "匹配成功占比:"+precent+"%" + def noMatchTotalCount = findNoMatchCount(sDate) println "第三方小区匹配还未匹配数量: "+noMatchTotalCount - saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount); + saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount); } @@ -59,11 +78,36 @@ class ThardParthAnalyBatch extends BaseBatch{ return count; } + def findAllEsDistDataCount(mdate){ + def count = null + ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d-> + count = d.count + }) + return count; + } + def findAllMongoDatasCount(s_date){ def q = new BasicDBObject().append("s_date", s_date) return thirdPartyColl.count(q) } + def findAllMongoDistrictCount(s_date){ + + def q = new BasicDBObject().append("s_date", s_date) + q.put("avg_price", new BasicDBObject('$gt', 0)) + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("region",1) + keys.put("name",1) + def dbcursor = thirdPartyColl.find(q,keys) + + def set = new HashSet() + for (def d : dbcursor) { + set << d.city+"##"+d.region+"##"+d.name + } + return set.size() + } + def findAllMongoDatasHavePricesCount(sDate){ def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)) return thirdPartyColl.count(q) @@ -74,19 +118,33 @@ class ThardParthAnalyBatch extends BaseBatch{ return thirdPartyColl.count(q) } + def findMatchSuccessDistCount(sDate){ + def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4) + def dbcursor = thirdPartyColl.count(q) + def set = new HashSet() + for (def d : dbcursor) { + set << d.city+"##"+d.region+"##"+d.name + } + return set.size() + } + def findNoMatchCount(sDate){ def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false)) return thirdPartyColl.count(q) } - def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){ + def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){ def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate)) def udata = new BasicDBObject() + udata.put("fjw_dist_total", fjwTotalCount) udata.put("crawl_total", crawlTotalCount) udata.put("synch_total", synchTotalCount) udata.put("synch_have_price_total", synchHavePriceTotalCount) + udata.put("synch_have_price_district_total", synchDistCount) udata.put("match_total", matchTotalCount) + udata.put("match_dist_total", matchDistTotalCount) + udata.put("match_dist_rate", precent) udata.put("no_match_total", noMatchTotalCount) udata.put("c_date", new Date()) if(obj){ From 6872caffc108afc0c09ee16e1f05886ebb3acafd Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 14 Aug 2024 14:24:13 +0800 Subject: [PATCH 3/5] update --- .../fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy index ee60d55b2..7453c70a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -120,7 +120,7 @@ class ThardParthAnalyBatch extends BaseBatch{ def findMatchSuccessDistCount(sDate){ def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4) - def dbcursor = thirdPartyColl.count(q) + def dbcursor = thirdPartyColl.find(q) def set = new HashSet() for (def d : dbcursor) { set << d.city+"##"+d.region+"##"+d.name From 6ed32bce1d6652666a7d04b09eb49b0eea8d20ce Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 14 Aug 2024 14:45:51 +0800 Subject: [PATCH 4/5] update --- .../fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy index 7453c70a7..855403723 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -37,7 +37,7 @@ class ThardParthAnalyBatch extends BaseBatch{ def run(){ def fjwTotalCount = CommonDatautils.findAllCount("district") - def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd") + def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); println "统计月份:"+sDate+" 查询更新时间:"+mdate From 022b26f67034462bca8c09514f3596df03a0b2d7 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 15 Aug 2024 09:59:04 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E6=97=B6=E9=97=B4=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy index 855403723..6a2ee9eec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -38,7 +38,7 @@ class ThardParthAnalyBatch extends BaseBatch{ def run(){ def fjwTotalCount = CommonDatautils.findAllCount("district") def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") - def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); println "统计月份:"+sDate+" 查询更新时间:"+mdate def crawlTotalCount = findAllEsDataCount(mdate)