update
This commit is contained in:
+6
@@ -1209,6 +1209,12 @@ class CommonDatautils {
|
||||
return rets;
|
||||
}
|
||||
|
||||
public static def findAllCount(String cat){
|
||||
DBObject q = new BasicDBObject();
|
||||
q.put("cat", cat);
|
||||
return SEAWEED_MONGO_COLL.count(q)
|
||||
}
|
||||
|
||||
|
||||
public static def findDistrictsByName(String city, String name, String cat){
|
||||
|
||||
|
||||
+22
-17
@@ -37,7 +37,7 @@ class DevProcess {
|
||||
def task = new DevProcess();
|
||||
// task.run()
|
||||
// task.parseCityAndDist()
|
||||
// task.devSplitRoad()
|
||||
task.devSplitRoad()
|
||||
// task.devNameSplitRoad()
|
||||
// task.addFlag()
|
||||
// task.matchDev()
|
||||
@@ -69,7 +69,7 @@ class DevProcess {
|
||||
|
||||
// task.addKfc()
|
||||
// task.devToPark() //开发区下辖园区
|
||||
task.checkTown()
|
||||
// task.checkTown()
|
||||
|
||||
}
|
||||
|
||||
@@ -349,22 +349,27 @@ class DevProcess {
|
||||
}
|
||||
|
||||
def matchDev(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\省级开发区名单\\新疆开发区名单.xlsx",{d->
|
||||
if(!d['Q'] && d['A']){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
|
||||
if(!d['I'] && d['J']){
|
||||
def q = new BasicDBObject()
|
||||
// q.put("name", d['A'])
|
||||
// q.put("name", d['J'])
|
||||
// q.put("city",d['G'])
|
||||
// q.put('alias', new BasicDBObject('$in',[d['A']]))
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+d['A']+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
// q.put('alias', new BasicDBObject('$in',[d['J']]))
|
||||
def name = d['J'].replace('经济技术开发区','经济开发区')
|
||||
q.put('name',new BasicDBObject('$regex', Pattern.compile('^.*'+name+'.*$', Pattern.CASE_INSENSITIVE))) //名称 模糊查询
|
||||
def obj = development_zone_CONN.findOne(q)
|
||||
if(obj){
|
||||
d['C'] = obj._id.toString()
|
||||
d['D'] = obj.name
|
||||
if(obj.name == d['A']){
|
||||
d['E'] = "相同"
|
||||
}else{
|
||||
d['E'] = "包含"
|
||||
}
|
||||
d['I'] = "库中有开发区-相似2"
|
||||
d['K'] = obj.name
|
||||
// d['C'] = obj._id.toString()
|
||||
// d['D'] = obj.name
|
||||
// if(obj.name == d['A']){
|
||||
// d['E'] = "相同"
|
||||
// }else{
|
||||
// d['E'] = "包含"
|
||||
// }
|
||||
// d['E'] = "别名"
|
||||
// d['F'] = obj.name
|
||||
}
|
||||
}
|
||||
},0);
|
||||
@@ -913,12 +918,12 @@ class DevProcess {
|
||||
|
||||
|
||||
def checkTown(){
|
||||
ExcelUtils.multiThreadingModify("D:\\数据\\数据建设\\开发区\\开发区下辖园区.xlsx",{d->
|
||||
ExcelUtils.modify("D:\\数据\\数据建设\\开发区\\前瞻网开发区对应园区.xlsx",{d->
|
||||
|
||||
if(!d['E']){
|
||||
if(d['C'] && d['D']){
|
||||
if(d['A']){
|
||||
// if(d['A']=="长春高新技术产业开发区"){
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['C'], d['D']+d['B'])
|
||||
def jsonObject1 = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(d['B'], d['A'])
|
||||
if(jsonObject1) {
|
||||
if (jsonObject1.get("townCode") && jsonObject1.get("townCode")!='null' && jsonObject1.get("townCode")!='') {
|
||||
def township = jsonObject1.get("township")
|
||||
|
||||
+63
-5
@@ -1,14 +1,19 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.AggregationOutput
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThardParthAnalyBatch extends BaseBatch{
|
||||
@@ -31,22 +36,36 @@ class ThardParthAnalyBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
def run(){
|
||||
def fjwTotalCount = CommonDatautils.findAllCount("district")
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -2), "yyyy-MM-dd")
|
||||
def sDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM"));
|
||||
println "统计月份:"+sDate+" 查询更新时间:"+mdate
|
||||
|
||||
def crawlTotalCount = findAllEsDataCount(mdate)
|
||||
println "抓取第三方小区量:"+crawlTotalCount
|
||||
println "抓取第三方小区总量:"+crawlTotalCount
|
||||
def synchTotalCount = findAllMongoDatasCount(sDate)
|
||||
println "同步到第三方库小区量: "+synchTotalCount
|
||||
println "同步到第三方库小区总量: "+synchTotalCount
|
||||
|
||||
def synchHavePriceTotalCount = findAllMongoDatasHavePricesCount(sDate)
|
||||
println "同步到第三方库有价格数量:"+synchHavePriceTotalCount
|
||||
|
||||
def synchDistCount = findAllMongoDistrictCount(sDate)
|
||||
println "同步到第三方库有价格小区量: "+synchDistCount
|
||||
|
||||
def matchTotalCount = findMatchSuccessCount(sDate)
|
||||
println "第三方小区匹配成功量:"+matchTotalCount
|
||||
println "第三方匹配成功总量:"+matchTotalCount
|
||||
|
||||
def matchDistTotalCount = findMatchSuccessDistCount(sDate)
|
||||
println "第三方匹配成功小区量:"+matchDistTotalCount
|
||||
|
||||
def precent = matchDistTotalCount*100.0/fjwTotalCount
|
||||
precent = DataHandleUtils.getDoubleRound(2, precent)
|
||||
println "匹配成功占比:"+precent+"%"
|
||||
|
||||
def noMatchTotalCount = findNoMatchCount(sDate)
|
||||
println "第三方小区匹配还未匹配数量: "+noMatchTotalCount
|
||||
|
||||
saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount);
|
||||
saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount,matchDistTotalCount, precent,noMatchTotalCount);
|
||||
|
||||
}
|
||||
|
||||
@@ -59,11 +78,36 @@ class ThardParthAnalyBatch extends BaseBatch{
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllEsDistDataCount(mdate){
|
||||
def count = null
|
||||
ElasticUtils.executeSqlQuery("select city,region,name as count from community where m_date>='${mdate}'", { d->
|
||||
count = d.count
|
||||
})
|
||||
return count;
|
||||
}
|
||||
|
||||
def findAllMongoDatasCount(s_date){
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findAllMongoDistrictCount(s_date){
|
||||
|
||||
def q = new BasicDBObject().append("s_date", s_date)
|
||||
q.put("avg_price", new BasicDBObject('$gt', 0))
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("city",1)
|
||||
keys.put("region",1)
|
||||
keys.put("name",1)
|
||||
def dbcursor = thirdPartyColl.find(q,keys)
|
||||
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findAllMongoDatasHavePricesCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0))
|
||||
return thirdPartyColl.count(q)
|
||||
@@ -74,19 +118,33 @@ class ThardParthAnalyBatch extends BaseBatch{
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def findMatchSuccessDistCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", 4)
|
||||
def dbcursor = thirdPartyColl.count(q)
|
||||
def set = new HashSet()
|
||||
for (def d : dbcursor) {
|
||||
set << d.city+"##"+d.region+"##"+d.name
|
||||
}
|
||||
return set.size()
|
||||
}
|
||||
|
||||
def findNoMatchCount(sDate){
|
||||
def q = new BasicDBObject().append("s_date", sDate).append("avg_price", new BasicDBObject('$gt', 0)).append("status", new BasicDBObject('$exists', false))
|
||||
return thirdPartyColl.count(q)
|
||||
}
|
||||
|
||||
def saveOrUpdateDatas(sDate, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, matchTotalCount, noMatchTotalCount){
|
||||
def saveOrUpdateDatas(sDate, fjwTotalCount, crawlTotalCount, synchTotalCount, synchHavePriceTotalCount, synchDistCount, matchTotalCount, matchDistTotalCount, precent, noMatchTotalCount){
|
||||
def obj = THIRD_PARTY_STAT.findOne(new BasicDBObject("s_date", sDate))
|
||||
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("fjw_dist_total", fjwTotalCount)
|
||||
udata.put("crawl_total", crawlTotalCount)
|
||||
udata.put("synch_total", synchTotalCount)
|
||||
udata.put("synch_have_price_total", synchHavePriceTotalCount)
|
||||
udata.put("synch_have_price_district_total", synchDistCount)
|
||||
udata.put("match_total", matchTotalCount)
|
||||
udata.put("match_dist_total", matchDistTotalCount)
|
||||
udata.put("match_dist_rate", precent)
|
||||
udata.put("no_match_total", noMatchTotalCount)
|
||||
udata.put("c_date", new Date())
|
||||
if(obj){
|
||||
|
||||
Reference in New Issue
Block a user