update
This commit is contained in:
+121
-120
@@ -52,7 +52,8 @@ class UpdateDistrictField extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
def margeResults = megerDatas(sourceMap)
|
def margeResults = megerDatas(sourceMap)
|
||||||
// updateDB(margeResults)
|
updateDB(margeResults)
|
||||||
|
println "修改完成 "+new Date().toString()
|
||||||
}
|
}
|
||||||
|
|
||||||
def megerDatas(sourceMap){
|
def megerDatas(sourceMap){
|
||||||
@@ -101,74 +102,74 @@ class UpdateDistrictField extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
|
||||||
def lianjisIds = lianjiaMap.keySet()
|
// def lianjisIds = lianjiaMap.keySet()
|
||||||
def wiwjIds = wiwjMap.keySet()
|
// def wiwjIds = wiwjMap.keySet()
|
||||||
def anjukeIds = anjukeMap.keySet()
|
// def anjukeIds = anjukeMap.keySet()
|
||||||
def set = new HashSet()
|
// def set = new HashSet()
|
||||||
set.addAll(lianjisIds)
|
// set.addAll(lianjisIds)
|
||||||
set.addAll(wiwjIds)
|
// set.addAll(wiwjIds)
|
||||||
set.addAll(anjukeIds)
|
// set.addAll(anjukeIds)
|
||||||
|
//
|
||||||
def lastValues = []
|
// def lastValues = []
|
||||||
def lianjiaUpdateCount = 0
|
// def lianjiaUpdateCount = 0
|
||||||
def wiwjUpdateCount = 0
|
// def wiwjUpdateCount = 0
|
||||||
def anjukeUpdateCount = 0
|
// def anjukeUpdateCount = 0
|
||||||
def lianjiaCountMap = [:]
|
// def lianjiaCountMap = [:]
|
||||||
def wiwjCountMap = [:]
|
// def wiwjCountMap = [:]
|
||||||
def anjukeCountMap = [:]
|
// def anjukeCountMap = [:]
|
||||||
set.each {id->
|
// set.each {id->
|
||||||
BasicDBObject newValMap = new BasicDBObject()
|
// BasicDBObject newValMap = new BasicDBObject()
|
||||||
fields.each {field->
|
// fields.each {field->
|
||||||
def lianjiaValMap = lianjiaMap.get(id)
|
// def lianjiaValMap = lianjiaMap.get(id)
|
||||||
def wiwjValMap = wiwjMap.get(id)
|
// def wiwjValMap = wiwjMap.get(id)
|
||||||
def anjukeValMap = anjukeMap.get(id)
|
// def anjukeValMap = anjukeMap.get(id)
|
||||||
if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
|
||||||
newValMap.put(field, lianjiaValMap.get(field))
|
// newValMap.put(field, lianjiaValMap.get(field))
|
||||||
lianjiaUpdateCount++
|
// lianjiaUpdateCount++
|
||||||
def count = lianjiaCountMap.get(field)
|
// def count = lianjiaCountMap.get(field)
|
||||||
if(!count){
|
// if(!count){
|
||||||
lianjiaCountMap.put(field, 1)
|
// lianjiaCountMap.put(field, 1)
|
||||||
}else{
|
// }else{
|
||||||
lianjiaCountMap.put(field, count + 1)
|
// lianjiaCountMap.put(field, count + 1)
|
||||||
}
|
// }
|
||||||
}else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
|
||||||
newValMap.put(field, wiwjValMap.get(field))
|
// newValMap.put(field, wiwjValMap.get(field))
|
||||||
wiwjUpdateCount++
|
// wiwjUpdateCount++
|
||||||
def count = wiwjCountMap.get(field)
|
// def count = wiwjCountMap.get(field)
|
||||||
if(!count){
|
// if(!count){
|
||||||
wiwjCountMap.put(field, 1)
|
// wiwjCountMap.put(field, 1)
|
||||||
}else{
|
// }else{
|
||||||
wiwjCountMap.put(field, count + 1)
|
// wiwjCountMap.put(field, count + 1)
|
||||||
}
|
// }
|
||||||
}else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
|
||||||
newValMap.put(field, anjukeValMap.get(field))
|
// newValMap.put(field, anjukeValMap.get(field))
|
||||||
anjukeUpdateCount++
|
// anjukeUpdateCount++
|
||||||
def count = anjukeCountMap.get(field)
|
// def count = anjukeCountMap.get(field)
|
||||||
if(!count){
|
// if(!count){
|
||||||
anjukeCountMap.put(field, 1)
|
// anjukeCountMap.put(field, 1)
|
||||||
}else{
|
// }else{
|
||||||
anjukeCountMap.put(field, count + 1)
|
// anjukeCountMap.put(field, count + 1)
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
newValMap.put("id", id)
|
// newValMap.put("id", id)
|
||||||
lastValues << newValMap
|
// lastValues << newValMap
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
lianjiaCountMap.each {k,v->
|
// lianjiaCountMap.each {k,v->
|
||||||
println "[链家] "+k+" "+v
|
// println "[链家] "+k+" "+v
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
wiwjCountMap.each {k,v->
|
// wiwjCountMap.each {k,v->
|
||||||
println "[我爱我家] "+k+" "+v
|
// println "[我爱我家] "+k+" "+v
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
anjukeCountMap.each {k,v->
|
// anjukeCountMap.each {k,v->
|
||||||
println "[安居客] "+k+" "+v
|
// println "[安居客] "+k+" "+v
|
||||||
}
|
// }
|
||||||
return lastValues
|
// return lastValues
|
||||||
}
|
// }
|
||||||
|
|
||||||
def updateDB(margeResults){
|
def updateDB(margeResults){
|
||||||
for(def mr : margeResults){
|
for(def mr : margeResults){
|
||||||
@@ -179,57 +180,57 @@ class UpdateDistrictField extends BaseBatch{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
def batchUpdateDist(source){
|
// def batchUpdateDist(source){
|
||||||
// def rets = findThirdMatchDist(source)
|
//// def rets = findThirdMatchDist(source)
|
||||||
println "[${source}] 符保条件小区量:"+rets.size()
|
// println "[${source}] 符保条件小区量:"+rets.size()
|
||||||
def updateIndex = new AtomicLong()
|
// def updateIndex = new AtomicLong()
|
||||||
final CountDownLatch latch = new CountDownLatch(rets.size())
|
// final CountDownLatch latch = new CountDownLatch(rets.size())
|
||||||
def allUpdateList = new CopyOnWriteArrayList()
|
// def allUpdateList = new CopyOnWriteArrayList()
|
||||||
|
//
|
||||||
def sourceMap = new ConcurrentHashMap()
|
// def sourceMap = new ConcurrentHashMap()
|
||||||
|
//
|
||||||
for(def i=0;i<rets.size();i++) {
|
// for(def i=0;i<rets.size();i++) {
|
||||||
final def ret = rets.get(i)
|
// final def ret = rets.get(i)
|
||||||
// fixedThreadPool.execute(
|
//// fixedThreadPool.execute(
|
||||||
// new Runnable() {
|
//// new Runnable() {
|
||||||
// void run() {
|
//// void run() {
|
||||||
try {
|
// try {
|
||||||
def newValMap = running(ret.url, ret.old_match_id)
|
// def newValMap = running(ret.url, ret.old_match_id)
|
||||||
if(newValMap){
|
// if(newValMap){
|
||||||
sourceMap.put(ret.old_match_id, newValMap)
|
// sourceMap.put(ret.old_match_id, newValMap)
|
||||||
allUpdateList << newValMap
|
// allUpdateList << newValMap
|
||||||
updateIndex.addAndGet(1)
|
// updateIndex.addAndGet(1)
|
||||||
}
|
// }
|
||||||
}catch (Exception e) {
|
// }catch (Exception e) {
|
||||||
e.printStackTrace()
|
// e.printStackTrace()
|
||||||
} finally {
|
// } finally {
|
||||||
latch.countDown()
|
// latch.countDown()
|
||||||
}
|
// }
|
||||||
// }
|
//// }
|
||||||
// }
|
//// }
|
||||||
// )
|
//// )
|
||||||
}
|
// }
|
||||||
latch.await()
|
// latch.await()
|
||||||
println "总修改量:"+updateIndex.get()
|
// println "总修改量:"+updateIndex.get()
|
||||||
def countMap = [:]
|
// def countMap = [:]
|
||||||
for(def map : allUpdateList){
|
// for(def map : allUpdateList){
|
||||||
map.each{k,v->
|
// map.each{k,v->
|
||||||
def count = countMap.get(k)
|
// def count = countMap.get(k)
|
||||||
if(!count){
|
// if(!count){
|
||||||
countMap.put(k, 1)
|
// countMap.put(k, 1)
|
||||||
}else{
|
// }else{
|
||||||
countMap.put(k, count+1)
|
// countMap.put(k, count+1)
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
countMap.each {k,v->
|
// countMap.each {k,v->
|
||||||
println k +" 更新量:"+v
|
// println k +" 更新量:"+v
|
||||||
}
|
// }
|
||||||
|
//
|
||||||
println "---------------------------------------------------------"
|
// println "---------------------------------------------------------"
|
||||||
return sourceMap
|
// return sourceMap
|
||||||
}
|
// }
|
||||||
|
|
||||||
def running(url, old_match_id){
|
def running(url, old_match_id){
|
||||||
def esObj = findCommunities(url)
|
def esObj = findCommunities(url)
|
||||||
|
|||||||
+28
-21
@@ -38,7 +38,7 @@ class AddDistImagesBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||||
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||||
def citys = CommonUtils.findAllCity()
|
def citys = CommonUtils.findAllCity()
|
||||||
citys.each {city->
|
citys.each {city->
|
||||||
@@ -48,7 +48,7 @@ class AddDistImagesBatch extends BaseBatch{
|
|||||||
matchCommunitys(communitys)
|
matchCommunitys(communitys)
|
||||||
}
|
}
|
||||||
|
|
||||||
// statImgCount()
|
statImgCount()
|
||||||
}
|
}
|
||||||
def findAllEsData2(city, mdate){
|
def findAllEsData2(city, mdate){
|
||||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||||
@@ -105,40 +105,47 @@ class AddDistImagesBatch extends BaseBatch{
|
|||||||
def total = communitys.size()
|
def total = communitys.size()
|
||||||
for(def i=0;i<communitys.size();i++){
|
for(def i=0;i<communitys.size();i++){
|
||||||
final def d = communitys[i]
|
final def d = communitys[i]
|
||||||
// fixedThreadPool.execute(new Runnable() {
|
fixedThreadPool.execute(new Runnable() {
|
||||||
// void run() {
|
void run() {
|
||||||
try{
|
try{
|
||||||
addCount.addAndGet(1)
|
addCount.addAndGet(1)
|
||||||
|
|
||||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
||||||
def obj = thirdSeaweedMappingColl.findOne(query)
|
def obj = thirdSeaweedMappingColl.findOne(query)
|
||||||
if(obj) {
|
if(obj) {
|
||||||
def vo = new BasicDBObject()
|
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
|
||||||
vo.put("city", obj.match_city)
|
if(obj.old_match_id && obj.match_id==obj.old_match_id){
|
||||||
vo.put("district", obj.match_district)
|
def vo = new BasicDBObject()
|
||||||
vo.put("name", obj.match_name)
|
vo.put("city", obj.match_city)
|
||||||
if(d.out_images){
|
vo.put("district", obj.match_district)
|
||||||
vo.put("out_pics", d.out_images)
|
vo.put("name", obj.match_name)
|
||||||
|
if(d.out_images && (!imageObj || (imageObj && !imageObj.out_pics))){
|
||||||
|
vo.put("out_pics", d.out_images)
|
||||||
|
}
|
||||||
|
if(d.house_type_images && (!imageObj || (imageObj && !imageObj.floorplan_pics))){
|
||||||
|
vo.put("floorplan_pics", d.house_type_images)
|
||||||
|
}
|
||||||
|
vo.put("m_date", new Date())
|
||||||
|
rets.add(vo)
|
||||||
|
if(vo.out_pics || vo.floorplan_pics){
|
||||||
|
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||||
|
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||||
|
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if(d.house_type_images){
|
|
||||||
vo.put("floorplan_pics", d.house_type_images)
|
|
||||||
}
|
|
||||||
vo.put("m_date", new Date())
|
|
||||||
rets.add(vo)
|
|
||||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
|
||||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
|
||||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
|
||||||
}else{
|
}else{
|
||||||
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
|
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
|
||||||
}
|
}
|
||||||
|
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
e.printStackTrace()
|
e.printStackTrace()
|
||||||
} finally {
|
} finally {
|
||||||
latch.countDown()
|
latch.countDown()
|
||||||
}
|
}
|
||||||
// }
|
}
|
||||||
// });
|
});
|
||||||
}
|
}
|
||||||
// latch.await()
|
latch.await()
|
||||||
return rets
|
return rets
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
+1
@@ -1062,6 +1062,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
q.put("s_date", SHORT_DATE);
|
q.put("s_date", SHORT_DATE);
|
||||||
|
|
||||||
DBObject udata2 = new BasicDBObject()
|
DBObject udata2 = new BasicDBObject()
|
||||||
|
udata2.put("match_id", district._id.toString());
|
||||||
udata2.put("match_city", district.city);
|
udata2.put("match_city", district.city);
|
||||||
udata2.put("match_region", district.region);
|
udata2.put("match_region", district.region);
|
||||||
udata2.put("match_name", district.name);
|
udata2.put("match_name", district.name);
|
||||||
|
|||||||
Reference in New Issue
Block a user