小区图片更新

This commit is contained in:
2024-10-22 15:14:41 +08:00
parent 2f8d203a70
commit 352c454b2c
@@ -0,0 +1,146 @@
package com.fangjia.script.gj.images
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 更新小区图片
*/
@BatchOwner(Watcher.GaoJuan)
class AddDistImagesBatch extends BaseBatch{
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com")
static main(args) {
runBatch(new AddDistImagesBatch(), args);
}
@Override
public void process(Object args) {
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
def citys = CommonUtils.findAllCity()
citys.each {city->
def communitys = findAllEsData2(city, mdate)
println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size()
//第三方匹配小区
matchCommunitys(communitys)
}
}
def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
.must(date_range).must(date_range)
def rets = []
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
if(d.house_type_images_2 || d.outdoor_images) {
def data = new BasicDBObject()
data.put("city", d.city)
data.put("region", d.region)
data.put("name", d.name)
data.put("id", d.id)
data.put("out_images", d.outdoor_images)
data.put("house_type_images", d.house_type_images_2)
rets << data
}
}
}
return rets;
}
def findAllEsData(city, mdate){
def names = new HashSet()
def rets = []
ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d->
if(d.house_type_images_2 || d.outdoor_images){
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
if(!names.contains(s)){
names << s
def data = new BasicDBObject()
data.put("city", d.city)
data.put("region", d.region)
data.put("name", d.name)
data.put("out_images", d.outdoor_images)
data.put("house_type_images", d.house_type_images_2)
rets << data
}
}
})
return rets;
}
def matchCommunitys(communitys){
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(communitys.size())
def rets = new CopyOnWriteArrayList()
def total = communitys.size()
for(def i=0;i<communitys.size();i++){
final def d = communitys[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
def obj = thirdSeaweedMappingColl.findOne(query)
if(obj) {
def vo = new BasicDBObject()
vo.put("city", obj.match_city)
vo.put("district", obj.match_district)
vo.put("name", obj.match_name)
if(d.out_images){
vo.put("out_pics", d.out_images)
}
if(d.house_type_images){
vo.put("floorplan_pics", d.house_type_images)
}
vo.put("m_date", new Date())
rets.add(vo)
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
println addCount.get()+"/"+total +" [匹配库中存在] "
}else{
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
latch.await()
return rets
}
}