Merge branch 'script_gj' into 'master'

update

See merge request !1072
This commit is contained in:
2025-02-07 17:00:21 +08:00
3 changed files with 150 additions and 141 deletions
@@ -52,7 +52,8 @@ class UpdateDistrictField extends BaseBatch{
}
}
def margeResults = megerDatas(sourceMap)
// updateDB(margeResults)
updateDB(margeResults)
println "修改完成 "+new Date().toString()
}
def megerDatas(sourceMap){
@@ -101,74 +102,74 @@ class UpdateDistrictField extends BaseBatch{
}
def margeValue(lianjiaMap, wiwjMap, anjukeMap){
def lianjisIds = lianjiaMap.keySet()
def wiwjIds = wiwjMap.keySet()
def anjukeIds = anjukeMap.keySet()
def set = new HashSet()
set.addAll(lianjisIds)
set.addAll(wiwjIds)
set.addAll(anjukeIds)
def lastValues = []
def lianjiaUpdateCount = 0
def wiwjUpdateCount = 0
def anjukeUpdateCount = 0
def lianjiaCountMap = [:]
def wiwjCountMap = [:]
def anjukeCountMap = [:]
set.each {id->
BasicDBObject newValMap = new BasicDBObject()
fields.each {field->
def lianjiaValMap = lianjiaMap.get(id)
def wiwjValMap = wiwjMap.get(id)
def anjukeValMap = anjukeMap.get(id)
if(lianjiaValMap && lianjiaValMap.containsKey(field)){
newValMap.put(field, lianjiaValMap.get(field))
lianjiaUpdateCount++
def count = lianjiaCountMap.get(field)
if(!count){
lianjiaCountMap.put(field, 1)
}else{
lianjiaCountMap.put(field, count + 1)
}
}else if(wiwjValMap && wiwjValMap.containsKey(field)){
newValMap.put(field, wiwjValMap.get(field))
wiwjUpdateCount++
def count = wiwjCountMap.get(field)
if(!count){
wiwjCountMap.put(field, 1)
}else{
wiwjCountMap.put(field, count + 1)
}
}else if(anjukeValMap && anjukeValMap.containsKey(field)){
newValMap.put(field, anjukeValMap.get(field))
anjukeUpdateCount++
def count = anjukeCountMap.get(field)
if(!count){
anjukeCountMap.put(field, 1)
}else{
anjukeCountMap.put(field, count + 1)
}
}
}
newValMap.put("id", id)
lastValues << newValMap
}
lianjiaCountMap.each {k,v->
println "[链家] "+k+" "+v
}
wiwjCountMap.each {k,v->
println "[我爱我家] "+k+" "+v
}
anjukeCountMap.each {k,v->
println "[安居客] "+k+" "+v
}
return lastValues
}
// def margeValue(lianjiaMap, wiwjMap, anjukeMap){
// def lianjisIds = lianjiaMap.keySet()
// def wiwjIds = wiwjMap.keySet()
// def anjukeIds = anjukeMap.keySet()
// def set = new HashSet()
// set.addAll(lianjisIds)
// set.addAll(wiwjIds)
// set.addAll(anjukeIds)
//
// def lastValues = []
// def lianjiaUpdateCount = 0
// def wiwjUpdateCount = 0
// def anjukeUpdateCount = 0
// def lianjiaCountMap = [:]
// def wiwjCountMap = [:]
// def anjukeCountMap = [:]
// set.each {id->
// BasicDBObject newValMap = new BasicDBObject()
// fields.each {field->
// def lianjiaValMap = lianjiaMap.get(id)
// def wiwjValMap = wiwjMap.get(id)
// def anjukeValMap = anjukeMap.get(id)
// if(lianjiaValMap && lianjiaValMap.containsKey(field)){
// newValMap.put(field, lianjiaValMap.get(field))
// lianjiaUpdateCount++
// def count = lianjiaCountMap.get(field)
// if(!count){
// lianjiaCountMap.put(field, 1)
// }else{
// lianjiaCountMap.put(field, count + 1)
// }
// }else if(wiwjValMap && wiwjValMap.containsKey(field)){
// newValMap.put(field, wiwjValMap.get(field))
// wiwjUpdateCount++
// def count = wiwjCountMap.get(field)
// if(!count){
// wiwjCountMap.put(field, 1)
// }else{
// wiwjCountMap.put(field, count + 1)
// }
// }else if(anjukeValMap && anjukeValMap.containsKey(field)){
// newValMap.put(field, anjukeValMap.get(field))
// anjukeUpdateCount++
// def count = anjukeCountMap.get(field)
// if(!count){
// anjukeCountMap.put(field, 1)
// }else{
// anjukeCountMap.put(field, count + 1)
// }
// }
// }
// newValMap.put("id", id)
// lastValues << newValMap
// }
//
// lianjiaCountMap.each {k,v->
// println "[链家] "+k+" "+v
// }
//
// wiwjCountMap.each {k,v->
// println "[我爱我家] "+k+" "+v
// }
//
// anjukeCountMap.each {k,v->
// println "[安居客] "+k+" "+v
// }
// return lastValues
// }
def updateDB(margeResults){
for(def mr : margeResults){
@@ -179,57 +180,57 @@ class UpdateDistrictField extends BaseBatch{
}
}
def batchUpdateDist(source){
// def rets = findThirdMatchDist(source)
println "[${source}] 符保条件小区量:"+rets.size()
def updateIndex = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(rets.size())
def allUpdateList = new CopyOnWriteArrayList()
def sourceMap = new ConcurrentHashMap()
for(def i=0;i<rets.size();i++) {
final def ret = rets.get(i)
// fixedThreadPool.execute(
// new Runnable() {
// void run() {
try {
def newValMap = running(ret.url, ret.old_match_id)
if(newValMap){
sourceMap.put(ret.old_match_id, newValMap)
allUpdateList << newValMap
updateIndex.addAndGet(1)
}
}catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// }
// )
}
latch.await()
println "总修改量:"+updateIndex.get()
def countMap = [:]
for(def map : allUpdateList){
map.each{k,v->
def count = countMap.get(k)
if(!count){
countMap.put(k, 1)
}else{
countMap.put(k, count+1)
}
}
}
countMap.each {k,v->
println k +" 更新量:"+v
}
println "---------------------------------------------------------"
return sourceMap
}
// def batchUpdateDist(source){
//// def rets = findThirdMatchDist(source)
// println "[${source}] 符保条件小区量:"+rets.size()
// def updateIndex = new AtomicLong()
// final CountDownLatch latch = new CountDownLatch(rets.size())
// def allUpdateList = new CopyOnWriteArrayList()
//
// def sourceMap = new ConcurrentHashMap()
//
// for(def i=0;i<rets.size();i++) {
// final def ret = rets.get(i)
//// fixedThreadPool.execute(
//// new Runnable() {
//// void run() {
// try {
// def newValMap = running(ret.url, ret.old_match_id)
// if(newValMap){
// sourceMap.put(ret.old_match_id, newValMap)
// allUpdateList << newValMap
// updateIndex.addAndGet(1)
// }
// }catch (Exception e) {
// e.printStackTrace()
// } finally {
// latch.countDown()
// }
//// }
//// }
//// )
// }
// latch.await()
// println "总修改量:"+updateIndex.get()
// def countMap = [:]
// for(def map : allUpdateList){
// map.each{k,v->
// def count = countMap.get(k)
// if(!count){
// countMap.put(k, 1)
// }else{
// countMap.put(k, count+1)
// }
// }
// }
//
// countMap.each {k,v->
// println k +" 更新量:"+v
// }
//
// println "---------------------------------------------------------"
// return sourceMap
// }
def running(url, old_match_id){
def esObj = findCommunities(url)
@@ -38,7 +38,7 @@ class AddDistImagesBatch extends BaseBatch{
@Override
public void process(Object args) {
def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
// def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def citys = CommonUtils.findAllCity()
citys.each {city->
@@ -48,7 +48,7 @@ class AddDistImagesBatch extends BaseBatch{
matchCommunitys(communitys)
}
// statImgCount()
statImgCount()
}
def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
@@ -105,40 +105,47 @@ class AddDistImagesBatch extends BaseBatch{
def total = communitys.size()
for(def i=0;i<communitys.size();i++){
final def d = communitys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
def obj = thirdSeaweedMappingColl.findOne(query)
if(obj) {
def vo = new BasicDBObject()
vo.put("city", obj.match_city)
vo.put("district", obj.match_district)
vo.put("name", obj.match_name)
if(d.out_images){
vo.put("out_pics", d.out_images)
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
if(obj.old_match_id && obj.match_id==obj.old_match_id){
def vo = new BasicDBObject()
vo.put("city", obj.match_city)
vo.put("district", obj.match_district)
vo.put("name", obj.match_name)
if(d.out_images && (!imageObj || (imageObj && !imageObj.out_pics))){
vo.put("out_pics", d.out_images)
}
if(d.house_type_images && (!imageObj || (imageObj && !imageObj.floorplan_pics))){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
if(vo.out_pics || vo.floorplan_pics){
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}
}
if(d.house_type_images){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}else{
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
});
}
// latch.await()
latch.await()
return rets
}
@@ -1062,6 +1062,7 @@ class DataStandardBatch extends BaseBatch{
q.put("s_date", SHORT_DATE);
DBObject udata2 = new BasicDBObject()
udata2.put("match_id", district._id.toString());
udata2.put("match_city", district.city);
udata2.put("match_region", district.region);
udata2.put("match_name", district.name);