From 352c454b2c14d98d92ab27c58ede818af9c32663 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 22 Oct 2024 15:14:41 +0800 Subject: [PATCH] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=9B=BE=E7=89=87=E6=9B=B4?= =?UTF-8?q?=E6=96=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/images/AddDistImagesBatch.groovy | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy new file mode 100644 index 000000000..9ba1fe9ca --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/images/AddDistImagesBatch.groovy @@ -0,0 +1,146 @@ +package com.fangjia.script.gj.images + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 更新小区图片 + */ +@BatchOwner(Watcher.GaoJuan) +class AddDistImagesBatch extends BaseBatch{ + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + public static final DBCollection seaweed_fs_coll = MongoUtils.getCollectionByName("fangjia_base", "seaweed_images", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new AddDistImagesBatch(), args); + } + + @Override + public void process(Object args) { +// def mdate = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(), -1), "yyyy-MM-dd") + def mdate = DateUtils.getFirstDayOfMonth(new Date(), -1) + def citys = CommonUtils.findAllCity() + citys.each {city-> + def communitys = findAllEsData2(city, mdate) + println "抓取城市: "+city +" 第三方小区有图片数:"+communitys.size() + //第三方匹配小区 + matchCommunitys(communitys) + } + } + + + + def findAllEsData2(city, mdate){ + QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序")) + .must(date_range).must(date_range) + + def rets = [] + def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder); + if(datas) { + for (int k = 0; k < datas.size(); k++) { + def d = datas[k]; + if(d.house_type_images_2 || d.outdoor_images) { + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("id", d.id) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + } + return rets; + } + + def findAllEsData(city, mdate){ + def names = new HashSet() + def rets = [] + ElasticUtils.executeSqlQuery("select * from community where source='安居客小程序' and city='${city}' and m_date>='${mdate}' order by m_date", { d-> + if(d.house_type_images_2 || d.outdoor_images){ + def s = d.city +"#"+d.region+"#"+d.name+"#"+d.property_type + if(!names.contains(s)){ + names << s + def data = new BasicDBObject() + data.put("city", d.city) + data.put("region", d.region) + data.put("name", d.name) + data.put("out_images", d.outdoor_images) + data.put("house_type_images", d.house_type_images_2) + rets << data + } + } + }) + return rets; + } + + + def matchCommunitys(communitys){ + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(communitys.size()) + def rets = new CopyOnWriteArrayList() + def total = communitys.size() + for(def i=0;i