Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-08-15 17:20:33 +08:00
4 changed files with 99 additions and 26 deletions
@@ -441,6 +441,8 @@ public class AddrSvcController {
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else if (!"2".equals(type)) { // 对私的不用齐梁
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
@@ -538,7 +540,8 @@ public class AddrSvcController {
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else if (!"2".equals(type)) { // 对私的不用齐梁
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
@@ -674,7 +677,8 @@ public class AddrSvcController {
//我们省级及以上有开发区,直接用我们的结果
if (result.containsKey("devDistrictName")) {
basicModel.setResult(result);
} else if (!"2".equals(type)) { // 对私的不用齐梁
basicModel.setResult(result);
} else {
try {
//如果是江苏省或者浙江省 调用齐梁的结果
@@ -1209,6 +1209,12 @@ class CommonDatautils {
return rets;
}
public static def findAllCount(String cat){
DBObject q = new BasicDBObject();
q.put("cat", cat);
return SEAWEED_MONGO_COLL.count(q)
}
public static def findDistrictsByName(String city, String name, String cat){
@@ -37,7 +37,7 @@ class DevProcess {
def task = new DevProcess();
// task.run()
// task.parseCityAndDist()
// task.devSplitRoad()
task.devSplitRoad()
// task.devNameSplitRoad()
// task.addFlag()
// task.matchDev()
@@ -69,7 +69,7 @@ class DevProcess {
// task.addKfc()
// task.devToPark() //开发区下辖园区
task.checkTown()
// task.checkTown()
}
@@ -349,22 +349,27 @@ class DevProcess {
}
def matchDev(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\新疆开发区名单.xlsx",{d->
if(!d['Q'] && d['A']){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
if(!d['I'] && d['J']){
def q = new BasicDBObject()
// q.put("name", d['A'])
// q.put("name", d['J'])
// q.put("city",d['G'])
// q.put('alias', new BasicDBObject('$in',[d['A']]))
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+d['A']+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
// q.put('alias', new BasicDBObject('$in',[d['J']]))
def name = d['J'].replace('经济技术开发区','经济开发区')
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
def obj = development_zone_CONN.findOne(q)
if(obj){
d['C'] = obj._id.toString()
d['D'] = obj.name
if(obj.name == d['A']){
d['E'] = "相同"
}else{
d['E'] = "包含"
}
d['I'] = "库中有开发区-相似2"
d['K'] = obj.name
// d['C'] = obj._id.toString()
// d['D'] = obj.name
// if(obj.name == d['A']){
// d['E'] = "相同"
// }else{
// d['E'] = "包含"
// }
// d['E'] = "别名"
// d['F'] = obj.name
}
}
},0);
@@ -913,12 +918,12 @@ class DevProcess {
def checkTown(){
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx",{d->
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
if(!d['E']){
if(d['C'] && d['D']){
if(d['A']){
// if(d['A']=="长春高新技术产业开发区"){
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A'])
if(jsonObject1) {
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
def township = jsonObject1.get("township")
@@ -1,14 +1,19 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.AggregationOutput
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
@BatchOwner(Watcher.GaoJuan)
class ThardParthAnalyBatch extends BaseBatch{
@@ -31,22 +36,36 @@ class ThardParthAnalyBatch extends BaseBatch{
}
def run(){
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd")
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
def fjwTotalCount = CommonDatautils.findAllCount("district")
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
println "统计月份:"+sDate+" 查询更新时间:"+mdate
def crawlTotalCount = findAllEsDataCount(mdate)
println "抓取第三方小区量:"+crawlTotalCount
println "抓取第三方小区总量:"+crawlTotalCount
def synchTotalCount = findAllMongoDatasCount(sDate)
println "同步到第三方库小区量: "+synchTotalCount
println "同步到第三方库小区总量: "+synchTotalCount
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
def synchDistCount = findAllMongoDistrictCount(sDate)
println "同步到第三方库有价格小区量: "+synchDistCount
def matchTotalCount = findMatchSuccessCount(sDate)
println "第三方小区匹配成功量:"+matchTotalCount
println "第三方匹配成功总量:"+matchTotalCount
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
println "第三方匹配成功小区量:"+matchDistTotalCount
def precent = matchDistTotalCount*100.0/fjwTotalCount
precent = DataHandleUtils.getDoubleRound(2, precent)
println "匹配成功占比:"+precent+"%"
def noMatchTotalCount = findNoMatchCount(sDate)
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount);
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
}
@@ -59,11 +78,36 @@ class ThardParthAnalyBatch extends BaseBatch{
return count;
}
def findAllEsDistDataCount(mdate){
def count = null
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
count = d.count
})
return count;
}
def findAllMongoDatasCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
return thirdPartyColl.count(q)
}
def findAllMongoDistrictCount(s_date){
def q = new BasicDBObject().append("s_date", s_date)
q.put("avg_price", new BasicDBObject('$gt', 0))
def keys = new BasicDBObject()
keys.put("city",1)
keys.put("region",1)
keys.put("name",1)
def dbcursor = thirdPartyColl.find(q,keys)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findAllMongoDatasHavePricesCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
return thirdPartyColl.count(q)
@@ -74,19 +118,33 @@ class ThardParthAnalyBatch extends BaseBatch{
return thirdPartyColl.count(q)
}
def findMatchSuccessDistCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
def dbcursor = thirdPartyColl.find(q)
def set = new HashSet()
for (def d : dbcursor) {
set << d.city+"##"+d.region+"##"+d.name
}
return set.size()
}
def findNoMatchCount(sDate){
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
return thirdPartyColl.count(q)
}
def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
def udata = new BasicDBObject()
udata.put("fjw_dist_total", fjwTotalCount)
udata.put("crawl_total", crawlTotalCount)
udata.put("synch_total", synchTotalCount)
udata.put("synch_have_price_total", synchHavePriceTotalCount)
udata.put("synch_have_price_district_total", synchDistCount)
udata.put("match_total", matchTotalCount)
udata.put("match_dist_total", matchDistTotalCount)
udata.put("match_dist_rate", precent)
udata.put("no_match_total", noMatchTotalCount)
udata.put("c_date", new Date())
if(obj){