This commit is contained in:
2025-02-07 16:58:05 +08:00
parent 4906378a52
commit c717f3cf6c
3 changed files with 150 additions and 141 deletions
@@ -52,7 +52,8 @@ class UpdateDistrictField extends BaseBatch{
} }
} }
def margeResults = megerDatas(sourceMap) def margeResults = megerDatas(sourceMap)
// updateDB(margeResults) updateDB(margeResults)
println "修改完成 "+new Date().toString()
} }
def megerDatas(sourceMap){ def megerDatas(sourceMap){
@@ -101,74 +102,74 @@ class UpdateDistrictField extends BaseBatch{
} }
def margeValue(lianjiaMap, wiwjMap, anjukeMap){ // def margeValue(lianjiaMap, wiwjMap, anjukeMap){
def lianjisIds = lianjiaMap.keySet() // def lianjisIds = lianjiaMap.keySet()
def wiwjIds = wiwjMap.keySet() // def wiwjIds = wiwjMap.keySet()
def anjukeIds = anjukeMap.keySet() // def anjukeIds = anjukeMap.keySet()
def set = new HashSet() // def set = new HashSet()
set.addAll(lianjisIds) // set.addAll(lianjisIds)
set.addAll(wiwjIds) // set.addAll(wiwjIds)
set.addAll(anjukeIds) // set.addAll(anjukeIds)
//
def lastValues = [] // def lastValues = []
def lianjiaUpdateCount = 0 // def lianjiaUpdateCount = 0
def wiwjUpdateCount = 0 // def wiwjUpdateCount = 0
def anjukeUpdateCount = 0 // def anjukeUpdateCount = 0
def lianjiaCountMap = [:] // def lianjiaCountMap = [:]
def wiwjCountMap = [:] // def wiwjCountMap = [:]
def anjukeCountMap = [:] // def anjukeCountMap = [:]
set.each {id-> // set.each {id->
BasicDBObject newValMap = new BasicDBObject() // BasicDBObject newValMap = new BasicDBObject()
fields.each {field-> // fields.each {field->
def lianjiaValMap = lianjiaMap.get(id) // def lianjiaValMap = lianjiaMap.get(id)
def wiwjValMap = wiwjMap.get(id) // def wiwjValMap = wiwjMap.get(id)
def anjukeValMap = anjukeMap.get(id) // def anjukeValMap = anjukeMap.get(id)
if(lianjiaValMap && lianjiaValMap.containsKey(field)){ // if(lianjiaValMap && lianjiaValMap.containsKey(field)){
newValMap.put(field, lianjiaValMap.get(field)) // newValMap.put(field, lianjiaValMap.get(field))
lianjiaUpdateCount++ // lianjiaUpdateCount++
def count = lianjiaCountMap.get(field) // def count = lianjiaCountMap.get(field)
if(!count){ // if(!count){
lianjiaCountMap.put(field, 1) // lianjiaCountMap.put(field, 1)
}else{ // }else{
lianjiaCountMap.put(field, count + 1) // lianjiaCountMap.put(field, count + 1)
} // }
}else if(wiwjValMap && wiwjValMap.containsKey(field)){ // }else if(wiwjValMap && wiwjValMap.containsKey(field)){
newValMap.put(field, wiwjValMap.get(field)) // newValMap.put(field, wiwjValMap.get(field))
wiwjUpdateCount++ // wiwjUpdateCount++
def count = wiwjCountMap.get(field) // def count = wiwjCountMap.get(field)
if(!count){ // if(!count){
wiwjCountMap.put(field, 1) // wiwjCountMap.put(field, 1)
}else{ // }else{
wiwjCountMap.put(field, count + 1) // wiwjCountMap.put(field, count + 1)
} // }
}else if(anjukeValMap && anjukeValMap.containsKey(field)){ // }else if(anjukeValMap && anjukeValMap.containsKey(field)){
newValMap.put(field, anjukeValMap.get(field)) // newValMap.put(field, anjukeValMap.get(field))
anjukeUpdateCount++ // anjukeUpdateCount++
def count = anjukeCountMap.get(field) // def count = anjukeCountMap.get(field)
if(!count){ // if(!count){
anjukeCountMap.put(field, 1) // anjukeCountMap.put(field, 1)
}else{ // }else{
anjukeCountMap.put(field, count + 1) // anjukeCountMap.put(field, count + 1)
} // }
} // }
} // }
newValMap.put("id", id) // newValMap.put("id", id)
lastValues << newValMap // lastValues << newValMap
} // }
//
lianjiaCountMap.each {k,v-> // lianjiaCountMap.each {k,v->
println "[链家] "+k+" "+v // println "[链家] "+k+" "+v
} // }
//
wiwjCountMap.each {k,v-> // wiwjCountMap.each {k,v->
println "[我爱我家] "+k+" "+v // println "[我爱我家] "+k+" "+v
} // }
//
anjukeCountMap.each {k,v-> // anjukeCountMap.each {k,v->
println "[安居客] "+k+" "+v // println "[安居客] "+k+" "+v
} // }
return lastValues // return lastValues
} // }
def updateDB(margeResults){ def updateDB(margeResults){
for(def mr : margeResults){ for(def mr : margeResults){
@@ -179,57 +180,57 @@ class UpdateDistrictField extends BaseBatch{
} }
} }
def batchUpdateDist(source){ // def batchUpdateDist(source){
// def rets = findThirdMatchDist(source) //// def rets = findThirdMatchDist(source)
println "[${source}] 符保条件小区量:"+rets.size() // println "[${source}] 符保条件小区量:"+rets.size()
def updateIndex = new AtomicLong() // def updateIndex = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size()) // final CountDownLatch latch = new CountDownLatch(rets.size())
def allUpdateList = new CopyOnWriteArrayList() // def allUpdateList = new CopyOnWriteArrayList()
//
def sourceMap = new ConcurrentHashMap() // def sourceMap = new ConcurrentHashMap()
//
for(def i=0;i<rets.size();i++) { // for(def i=0;i<rets.size();i++) {
final def ret = rets.get(i) // final def ret = rets.get(i)
// fixedThreadPool.execute( //// fixedThreadPool.execute(
// new Runnable() { //// new Runnable() {
// void run() { //// void run() {
try { // try {
def newValMap = running(ret.url, ret.old_match_id) // def newValMap = running(ret.url, ret.old_match_id)
if(newValMap){ // if(newValMap){
sourceMap.put(ret.old_match_id, newValMap) // sourceMap.put(ret.old_match_id, newValMap)
allUpdateList << newValMap // allUpdateList << newValMap
updateIndex.addAndGet(1) // updateIndex.addAndGet(1)
} // }
}catch (Exception e) { // }catch (Exception e) {
e.printStackTrace() // e.printStackTrace()
} finally { // } finally {
latch.countDown() // latch.countDown()
} // }
// } //// }
// } //// }
// ) //// )
} // }
latch.await() // latch.await()
println "总修改量:"+updateIndex.get() // println "总修改量:"+updateIndex.get()
def countMap = [:] // def countMap = [:]
for(def map : allUpdateList){ // for(def map : allUpdateList){
map.each{k,v-> // map.each{k,v->
def count = countMap.get(k) // def count = countMap.get(k)
if(!count){ // if(!count){
countMap.put(k, 1) // countMap.put(k, 1)
}else{ // }else{
countMap.put(k, count+1) // countMap.put(k, count+1)
} // }
} // }
} // }
//
countMap.each {k,v-> // countMap.each {k,v->
println k +" 更新量:"+v // println k +" 更新量:"+v
} // }
//
println "---------------------------------------------------------" // println "---------------------------------------------------------"
return sourceMap // return sourceMap
} // }
def running(url, old_match_id){ def running(url, old_match_id){
def esObj = findCommunities(url) def esObj = findCommunities(url)
@@ -38,7 +38,7 @@ class AddDistImagesBatch extends BaseBatch{
@Override @Override
public void process(Object args) { public void process(Object args) {
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1) // def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def citys = CommonUtils.findAllCity() def citys = CommonUtils.findAllCity()
citys.each {city-> citys.each {city->
@@ -48,7 +48,7 @@ class AddDistImagesBatch extends BaseBatch{
matchCommunitys(communitys) matchCommunitys(communitys)
} }
// statImgCount() statImgCount()
} }
def findAllEsData2(city, mdate){ def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true) QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
@@ -105,40 +105,47 @@ class AddDistImagesBatch extends BaseBatch{
def total = communitys.size() def total = communitys.size()
for(def i=0;i<communitys.size();i++){ for(def i=0;i<communitys.size();i++){
final def d = communitys[i] final def d = communitys[i]
// fixedThreadPool.execute(new Runnable() { fixedThreadPool.execute(new Runnable() {
// void run() { void run() {
try{ try{
addCount.addAndGet(1) addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true)) def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
def obj = thirdSeaweedMappingColl.findOne(query) def obj = thirdSeaweedMappingColl.findOne(query)
if(obj) { if(obj) {
def vo = new BasicDBObject() def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
vo.put("city", obj.match_city) if(obj.old_match_id && obj.match_id==obj.old_match_id){
vo.put("district", obj.match_district) def vo = new BasicDBObject()
vo.put("name", obj.match_name) vo.put("city", obj.match_city)
if(d.out_images){ vo.put("district", obj.match_district)
vo.put("out_pics", d.out_images) vo.put("name", obj.match_name)
if(d.out_images && (!imageObj || (imageObj && !imageObj.out_pics))){
vo.put("out_pics", d.out_images)
}
if(d.house_type_images && (!imageObj || (imageObj && !imageObj.floorplan_pics))){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
if(vo.out_pics || vo.floorplan_pics){
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}
} }
if(d.house_type_images){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}else{ }else{
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString() println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
} }
} catch (Exception e) { } catch (Exception e) {
e.printStackTrace() e.printStackTrace()
} finally { } finally {
latch.countDown() latch.countDown()
} }
// } }
// }); });
} }
// latch.await() latch.await()
return rets return rets
} }
@@ -1062,6 +1062,7 @@ class DataStandardBatch extends BaseBatch{
q.put("s_date", SHORT_DATE); q.put("s_date", SHORT_DATE);
DBObject udata2 = new BasicDBObject() DBObject udata2 = new BasicDBObject()
udata2.put("match_id", district._id.toString());
udata2.put("match_city", district.city); udata2.put("match_city", district.city);
udata2.put("match_region", district.region); udata2.put("match_region", district.region);
udata2.put("match_name", district.name); udata2.put("match_name", district.name);