小区图片更新
This commit is contained in:
+146
@@ -0,0 +1,146 @@
|
||||
package com.fangjia.script.gj.images
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 更新小区图片
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class AddDistImagesBatch extends BaseBatch{
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com")
|
||||
|
||||
static main(args) {
|
||||
runBatch(new AddDistImagesBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd")
|
||||
def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1)
|
||||
def citys = CommonUtils.findAllCity()
|
||||
citys.each {city->
|
||||
def communitys = findAllEsData2(city, mdate)
|
||||
println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size()
|
||||
//第三方匹配小区
|
||||
matchCommunitys(communitys)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findAllEsData2(city, mdate){
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
|
||||
.must(date_range).must(date_range)
|
||||
|
||||
def rets = []
|
||||
def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||
if(datas) {
|
||||
for (int k = 0; k < datas.size(); k++) {
|
||||
def d = datas[k];
|
||||
if(d.house_type_images_2 || d.outdoor_images) {
|
||||
def data = new BasicDBObject()
|
||||
data.put("city", d.city)
|
||||
data.put("region", d.region)
|
||||
data.put("name", d.name)
|
||||
data.put("id", d.id)
|
||||
data.put("out_images", d.outdoor_images)
|
||||
data.put("house_type_images", d.house_type_images_2)
|
||||
rets << data
|
||||
}
|
||||
}
|
||||
}
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllEsData(city, mdate){
|
||||
def names = new HashSet()
|
||||
def rets = []
|
||||
ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d->
|
||||
if(d.house_type_images_2 || d.outdoor_images){
|
||||
def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type
|
||||
if(!names.contains(s)){
|
||||
names << s
|
||||
def data = new BasicDBObject()
|
||||
data.put("city", d.city)
|
||||
data.put("region", d.region)
|
||||
data.put("name", d.name)
|
||||
data.put("out_images", d.outdoor_images)
|
||||
data.put("house_type_images", d.house_type_images_2)
|
||||
rets << data
|
||||
}
|
||||
}
|
||||
})
|
||||
return rets;
|
||||
}
|
||||
|
||||
|
||||
def matchCommunitys(communitys){
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(communitys.size())
|
||||
def rets = new CopyOnWriteArrayList()
|
||||
def total = communitys.size()
|
||||
for(def i=0;i<communitys.size();i++){
|
||||
final def d = communitys[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
||||
def obj = thirdSeaweedMappingColl.findOne(query)
|
||||
if(obj) {
|
||||
def vo = new BasicDBObject()
|
||||
vo.put("city", obj.match_city)
|
||||
vo.put("district", obj.match_district)
|
||||
vo.put("name", obj.match_name)
|
||||
if(d.out_images){
|
||||
vo.put("out_pics", d.out_images)
|
||||
}
|
||||
if(d.house_type_images){
|
||||
vo.put("floorplan_pics", d.house_type_images)
|
||||
}
|
||||
vo.put("m_date", new Date())
|
||||
rets.add(vo)
|
||||
def q = new BasicDBObject("_id", new ObjectId(obj.match_id))
|
||||
seaweed_fs_coll.update(q, new BasicDBObject('$set', vo), true,false)
|
||||
println addCount.get()+"/"+total +" [匹配库中存在] "
|
||||
}else{
|
||||
println addCount.get()+"/"+total +" [没找到相应的匹配数据] "+d._id.toString()
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// }
|
||||
// });
|
||||
}
|
||||
latch.await()
|
||||
return rets
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user