From e52540856f03f7dcae5137a0cf81bc8ec886a7f0 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 14 Aug 2024 14:14:25 +0800 Subject: [PATCH] update --- .../script/commData/CommonDatautils.groovy | 6 ++ .../script/gj/data_handle/DevProcess.groovy | 39 ++++++----- .../data_handle/ThardParthAnalyBatch.groovy | 68 +++++++++++++++++-- 3 files changed, 91 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index f197f974c..d36b853dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1209,6 +1209,12 @@ class CommonDatautils { return rets; } + public static def findAllCount(String cat){ + DBObject q = new BasicDBObject(); + q.put("cat", cat); + return SEAWEED_MONGO_COLL.count(q) + } + public static def findDistrictsByName(String city, String name, String cat){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy index f90ed42f6..772bbaf62 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevProcess.groovy @@ -37,7 +37,7 @@ class DevProcess { def task = new DevProcess(); // task.run() // task.parseCityAndDist() -// task.devSplitRoad() + task.devSplitRoad() // task.devNameSplitRoad() // task.addFlag() // task.matchDev() @@ -69,7 +69,7 @@ class DevProcess { // task.addKfc() // task.devToPark() //开发区下辖园区 - task.checkTown() +// task.checkTown() } @@ -349,22 +349,27 @@ class DevProcess { } def matchDev(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\新疆开发区名单.xlsx",{d-> - if(!d['Q'] && d['A']){ + ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> + if(!d['I'] && d['J']){ def q = new BasicDBObject() -// q.put("name", d['A']) +// q.put("name", d['J']) // q.put("city",d['G']) -// q.put('alias', new BasicDBObject('$in',[d['A']])) - q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+d['A']+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 +// q.put('alias', new BasicDBObject('$in',[d['J']])) + def name = d['J'].replace('经济技术开发区','经济开发区') + q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询 def obj = development_zone_CONN.findOne(q) if(obj){ - d['C'] = obj._id.toString() - d['D'] = obj.name - if(obj.name == d['A']){ - d['E'] = "相同" - }else{ - d['E'] = "包含" - } + d['I'] = "库中有开发区-相似2" + d['K'] = obj.name +// d['C'] = obj._id.toString() +// d['D'] = obj.name +// if(obj.name == d['A']){ +// d['E'] = "相同" +// }else{ +// d['E'] = "包含" +// } +// d['E'] = "别名" +// d['F'] = obj.name } } },0); @@ -913,12 +918,12 @@ class DevProcess { def checkTown(){ - ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx",{d-> + ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d-> if(!d['E']){ - if(d['C'] && d['D']){ + if(d['A']){ // if(d['A']=="长春高新技术产业开发区"){ - def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B']) + def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A']) if(jsonObject1) { if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') { def township = jsonObject1.get("township") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy index ef6476d48..ee60d55b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThardParthAnalyBatch.groovy @@ -1,14 +1,19 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.datastore.driver.FacetResult import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.MongoUtils +import com.mongodb.AggregationOutput import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject @BatchOwner(Watcher.GaoJuan) class ThardParthAnalyBatch extends BaseBatch{ @@ -31,22 +36,36 @@ class ThardParthAnalyBatch extends BaseBatch{ } def run(){ + def fjwTotalCount = CommonDatautils.findAllCount("district") def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd") def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); println "统计月份:"+sDate+" 查询更新时间:"+mdate def crawlTotalCount = findAllEsDataCount(mdate) - println "抓取第三方小区量:"+crawlTotalCount + println "抓取第三方小区总量:"+crawlTotalCount def synchTotalCount = findAllMongoDatasCount(sDate) - println "同步到第三方库小区量: "+synchTotalCount + println "同步到第三方库小区总量: "+synchTotalCount + def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate) println "同步到第三方库有价格数量:"+synchHavePriceTotalCount + + def synchDistCount = findAllMongoDistrictCount(sDate) + println "同步到第三方库有价格小区量: "+synchDistCount + def matchTotalCount = findMatchSuccessCount(sDate) - println "第三方小区匹配成功量:"+matchTotalCount + println "第三方匹配成功总量:"+matchTotalCount + + def matchDistTotalCount = findMatchSuccessDistCount(sDate) + println "第三方匹配成功小区量:"+matchDistTotalCount + + def precent = matchDistTotalCount*100.0/fjwTotalCount + precent = DataHandleUtils.getDoubleRound(2, precent) + println "匹配成功占比:"+precent+"%" + def noMatchTotalCount = findNoMatchCount(sDate) println "第三方小区匹配还未匹配数量: "+noMatchTotalCount - saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount); + saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount); } @@ -59,11 +78,36 @@ class ThardParthAnalyBatch extends BaseBatch{ return count; } + def findAllEsDistDataCount(mdate){ + def count = null + ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d-> + count = d.count + }) + return count; + } + def findAllMongoDatasCount(s_date){ def q = new BasicDBObject().append("s_date", s_date) return thirdPartyColl.count(q) } + def findAllMongoDistrictCount(s_date){ + + def q = new BasicDBObject().append("s_date", s_date) + q.put("avg_price", new BasicDBObject('$gt', 0)) + def keys = new BasicDBObject() + keys.put("city",1) + keys.put("region",1) + keys.put("name",1) + def dbcursor = thirdPartyColl.find(q,keys) + + def set = new HashSet() + for (def d : dbcursor) { + set << d.city+"##"+d.region+"##"+d.name + } + return set.size() + } + def findAllMongoDatasHavePricesCount(sDate){ def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)) return thirdPartyColl.count(q) @@ -74,19 +118,33 @@ class ThardParthAnalyBatch extends BaseBatch{ return thirdPartyColl.count(q) } + def findMatchSuccessDistCount(sDate){ + def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4) + def dbcursor = thirdPartyColl.count(q) + def set = new HashSet() + for (def d : dbcursor) { + set << d.city+"##"+d.region+"##"+d.name + } + return set.size() + } + def findNoMatchCount(sDate){ def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false)) return thirdPartyColl.count(q) } - def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){ + def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){ def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate)) def udata = new BasicDBObject() + udata.put("fjw_dist_total", fjwTotalCount) udata.put("crawl_total", crawlTotalCount) udata.put("synch_total", synchTotalCount) udata.put("synch_have_price_total", synchHavePriceTotalCount) + udata.put("synch_have_price_district_total", synchDistCount) udata.put("match_total", matchTotalCount) + udata.put("match_dist_total", matchDistTotalCount) + udata.put("match_dist_rate", precent) udata.put("no_match_total", noMatchTotalCount) udata.put("c_date", new Date()) if(obj){