update
This commit is contained in:
+121
-120
@@ -52,7 +52,8 @@ class UpdateDistrictField extends BaseBatch{
|
||||
}
|
||||
}
|
||||
def margeResults = megerDatas(sourceMap)
|
||||
// updateDB(margeResults)
|
||||
updateDB(margeResults)
|
||||
println "修改完成 "+new Date().toString()
|
||||
}
|
||||
|
||||
def megerDatas(sourceMap){
|
||||
@@ -101,74 +102,74 @@ class UpdateDistrictField extends BaseBatch{
|
||||
}
|
||||
|
||||
|
||||
def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||
def lianjisIds = lianjiaMap.keySet()
|
||||
def wiwjIds = wiwjMap.keySet()
|
||||
def anjukeIds = anjukeMap.keySet()
|
||||
def set = new HashSet()
|
||||
set.addAll(lianjisIds)
|
||||
set.addAll(wiwjIds)
|
||||
set.addAll(anjukeIds)
|
||||
|
||||
def lastValues = []
|
||||
def lianjiaUpdateCount = 0
|
||||
def wiwjUpdateCount = 0
|
||||
def anjukeUpdateCount = 0
|
||||
def lianjiaCountMap = [:]
|
||||
def wiwjCountMap = [:]
|
||||
def anjukeCountMap = [:]
|
||||
set.each {id->
|
||||
BasicDBObject newValMap = new BasicDBObject()
|
||||
fields.each {field->
|
||||
def lianjiaValMap = lianjiaMap.get(id)
|
||||
def wiwjValMap = wiwjMap.get(id)
|
||||
def anjukeValMap = anjukeMap.get(id)
|
||||
if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||
newValMap.put(field, lianjiaValMap.get(field))
|
||||
lianjiaUpdateCount++
|
||||
def count = lianjiaCountMap.get(field)
|
||||
if(!count){
|
||||
lianjiaCountMap.put(field, 1)
|
||||
}else{
|
||||
lianjiaCountMap.put(field, count + 1)
|
||||
}
|
||||
}else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||
newValMap.put(field, wiwjValMap.get(field))
|
||||
wiwjUpdateCount++
|
||||
def count = wiwjCountMap.get(field)
|
||||
if(!count){
|
||||
wiwjCountMap.put(field, 1)
|
||||
}else{
|
||||
wiwjCountMap.put(field, count + 1)
|
||||
}
|
||||
}else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||
newValMap.put(field, anjukeValMap.get(field))
|
||||
anjukeUpdateCount++
|
||||
def count = anjukeCountMap.get(field)
|
||||
if(!count){
|
||||
anjukeCountMap.put(field, 1)
|
||||
}else{
|
||||
anjukeCountMap.put(field, count + 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
newValMap.put("id", id)
|
||||
lastValues << newValMap
|
||||
}
|
||||
|
||||
lianjiaCountMap.each {k,v->
|
||||
println "[链家] "+k+" "+v
|
||||
}
|
||||
|
||||
wiwjCountMap.each {k,v->
|
||||
println "[我爱我家] "+k+" "+v
|
||||
}
|
||||
|
||||
anjukeCountMap.each {k,v->
|
||||
println "[安居客] "+k+" "+v
|
||||
}
|
||||
return lastValues
|
||||
}
|
||||
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||
// def lianjisIds = lianjiaMap.keySet()
|
||||
// def wiwjIds = wiwjMap.keySet()
|
||||
// def anjukeIds = anjukeMap.keySet()
|
||||
// def set = new HashSet()
|
||||
// set.addAll(lianjisIds)
|
||||
// set.addAll(wiwjIds)
|
||||
// set.addAll(anjukeIds)
|
||||
//
|
||||
// def lastValues = []
|
||||
// def lianjiaUpdateCount = 0
|
||||
// def wiwjUpdateCount = 0
|
||||
// def anjukeUpdateCount = 0
|
||||
// def lianjiaCountMap = [:]
|
||||
// def wiwjCountMap = [:]
|
||||
// def anjukeCountMap = [:]
|
||||
// set.each {id->
|
||||
// BasicDBObject newValMap = new BasicDBObject()
|
||||
// fields.each {field->
|
||||
// def lianjiaValMap = lianjiaMap.get(id)
|
||||
// def wiwjValMap = wiwjMap.get(id)
|
||||
// def anjukeValMap = anjukeMap.get(id)
|
||||
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||
// newValMap.put(field, lianjiaValMap.get(field))
|
||||
// lianjiaUpdateCount++
|
||||
// def count = lianjiaCountMap.get(field)
|
||||
// if(!count){
|
||||
// lianjiaCountMap.put(field, 1)
|
||||
// }else{
|
||||
// lianjiaCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||
// newValMap.put(field, wiwjValMap.get(field))
|
||||
// wiwjUpdateCount++
|
||||
// def count = wiwjCountMap.get(field)
|
||||
// if(!count){
|
||||
// wiwjCountMap.put(field, 1)
|
||||
// }else{
|
||||
// wiwjCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||
// newValMap.put(field, anjukeValMap.get(field))
|
||||
// anjukeUpdateCount++
|
||||
// def count = anjukeCountMap.get(field)
|
||||
// if(!count){
|
||||
// anjukeCountMap.put(field, 1)
|
||||
// }else{
|
||||
// anjukeCountMap.put(field, count + 1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// newValMap.put("id", id)
|
||||
// lastValues << newValMap
|
||||
// }
|
||||
//
|
||||
// lianjiaCountMap.each {k,v->
|
||||
// println "[链家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// wiwjCountMap.each {k,v->
|
||||
// println "[我爱我家] "+k+" "+v
|
||||
// }
|
||||
//
|
||||
// anjukeCountMap.each {k,v->
|
||||
// println "[安居客] "+k+" "+v
|
||||
// }
|
||||
// return lastValues
|
||||
// }
|
||||
|
||||
def updateDB(margeResults){
|
||||
for(def mr : margeResults){
|
||||
@@ -179,57 +180,57 @@ class UpdateDistrictField extends BaseBatch{
|
||||
}
|
||||
}
|
||||
|
||||
def batchUpdateDist(source){
|
||||
// def rets = findThirdMatchDist(source)
|
||||
println "[${source}] 符保条件小区量:"+rets.size()
|
||||
def updateIndex = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
def allUpdateList = new CopyOnWriteArrayList()
|
||||
|
||||
def sourceMap = new ConcurrentHashMap()
|
||||
|
||||
for(def i=0;i<rets.size();i++) {
|
||||
final def ret = rets.get(i)
|
||||
// fixedThreadPool.execute(
|
||||
// new Runnable() {
|
||||
// void run() {
|
||||
try {
|
||||
def newValMap = running(ret.url, ret.old_match_id)
|
||||
if(newValMap){
|
||||
sourceMap.put(ret.old_match_id, newValMap)
|
||||
allUpdateList << newValMap
|
||||
updateIndex.addAndGet(1)
|
||||
}
|
||||
}catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// }
|
||||
// )
|
||||
}
|
||||
latch.await()
|
||||
println "总修改量:"+updateIndex.get()
|
||||
def countMap = [:]
|
||||
for(def map : allUpdateList){
|
||||
map.each{k,v->
|
||||
def count = countMap.get(k)
|
||||
if(!count){
|
||||
countMap.put(k, 1)
|
||||
}else{
|
||||
countMap.put(k, count+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
countMap.each {k,v->
|
||||
println k +" 更新量:"+v
|
||||
}
|
||||
|
||||
println "---------------------------------------------------------"
|
||||
return sourceMap
|
||||
}
|
||||
// def batchUpdateDist(source){
|
||||
//// def rets = findThirdMatchDist(source)
|
||||
// println "[${source}] 符保条件小区量:"+rets.size()
|
||||
// def updateIndex = new AtomicLong()
|
||||
// final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||
// def allUpdateList = new CopyOnWriteArrayList()
|
||||
//
|
||||
// def sourceMap = new ConcurrentHashMap()
|
||||
//
|
||||
// for(def i=0;i<rets.size();i++) {
|
||||
// final def ret = rets.get(i)
|
||||
//// fixedThreadPool.execute(
|
||||
//// new Runnable() {
|
||||
//// void run() {
|
||||
// try {
|
||||
// def newValMap = running(ret.url, ret.old_match_id)
|
||||
// if(newValMap){
|
||||
// sourceMap.put(ret.old_match_id, newValMap)
|
||||
// allUpdateList << newValMap
|
||||
// updateIndex.addAndGet(1)
|
||||
// }
|
||||
// }catch (Exception e) {
|
||||
// e.printStackTrace()
|
||||
// } finally {
|
||||
// latch.countDown()
|
||||
// }
|
||||
//// }
|
||||
//// }
|
||||
//// )
|
||||
// }
|
||||
// latch.await()
|
||||
// println "总修改量:"+updateIndex.get()
|
||||
// def countMap = [:]
|
||||
// for(def map : allUpdateList){
|
||||
// map.each{k,v->
|
||||
// def count = countMap.get(k)
|
||||
// if(!count){
|
||||
// countMap.put(k, 1)
|
||||
// }else{
|
||||
// countMap.put(k, count+1)
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// countMap.each {k,v->
|
||||
// println k +" 更新量:"+v
|
||||
// }
|
||||
//
|
||||
// println "---------------------------------------------------------"
|
||||
// return sourceMap
|
||||
// }
|
||||
|
||||
def running(url, old_match_id){
|
||||
def esObj = findCommunities(url)
|
||||
|
||||
+28
-21
@@ -38,7 +38,7 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
def citys = CommonUtils.findAllCity()
|
||||
citys.each {city->
|
||||
@@ -48,7 +48,7 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
matchCommunitys(communitys)
|
||||
}
|
||||
|
||||
// statImgCount()
|
||||
statImgCount()
|
||||
}
|
||||
def findAllEsData2(city, mdate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||
@@ -105,40 +105,47 @@ class AddDistImagesBatch extends BaseBatch{
|
||||
def total = communitys.size()
|
||||
for(def i=0;i<communitys.size();i++){
|
||||
final def d = communitys[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
|
||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
||||
def obj = thirdSeaweedMappingColl.findOne(query)
|
||||
if(obj) {
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images){
|
||||
vo.put("out_pics", d.out_images)
|
||||
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
|
||||
if(obj.old_match_id && obj.match_id==obj.old_match_id){
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images && (!imageObj || (imageObj && !imageObj.out_pics))){
|
||||
vo.put("out_pics", d.out_images)
|
||||
}
|
||||
if(d.house_type_images && (!imageObj || (imageObj && !imageObj.floorplan_pics))){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
if(vo.out_pics || vo.floorplan_pics){
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}
|
||||
}
|
||||
if(d.house_type_images){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}else{
|
||||
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
});
|
||||
}
|
||||
// latch.await()
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
|
||||
+1
@@ -1062,6 +1062,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
q.put("s_date", SHORT_DATE);
|
||||
|
||||
DBObject udata2 = new BasicDBObject()
|
||||
udata2.put("match_id", district._id.toString());
|
||||
udata2.put("match_city", district.city);
|
||||
udata2.put("match_region", district.region);
|
||||
udata2.put("match_name", district.name);
|
||||
|
||||
Reference in New Issue
Block a user