diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy new file mode 100644 index 000000000..2d7515aa7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/ThirdPriceMappingBatch.groovy @@ -0,0 +1,214 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 第三方小区与我们小区映射关系表 + */ +@BatchOwner(Watcher.GaoJuan) +class ThirdPriceMappingBatch extends BaseBatch{ + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) + + + final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018) + + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + + static main(args) { + runBatch(new ThirdPriceMappingBatch(), args); + } + + @Override + public void process(Object args) { +// //将新增数据同步过来 +// addNemCommunity() +// //接口进行匹配 + match() + +// addOldMatchResult() + } + + def match(){ + def citys = CommonUtils.findAllCity() + for(def c : citys){ + def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false)) + def noMatchDatas = thirdSeaweedMappingColl.find(query).toList() + matchByCity(c, noMatchDatas) + } + + } + + def matchByCity(def c, def noMatchDatas){ + def totalCount = new AtomicLong() + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(noMatchDatas.size()) + def total = noMatchDatas.size() + for(def i=0;i + udata.put(k,v) + } + udata.put("m_date", new Date()) + thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata)) + }else{ + sd.put("m_date", new Date()) + thirdSeaweedMappingColl.save(sd) + addCount.addAndGet(1) + } + + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + } + }); + } + latch.await() + return addCount.get(); + } + + def addOldMatchResult(){ + def citys = ["上海"]//CommonUtils.findAllCity() + def keys = new BasicDBObject() + keys.put("_id", 1) + + for(def c : citys){ + def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList() + println c +" 待关联数据:"+datas.size() + findResultFromDB(c, datas) + println c + " 关联数据结束 "+DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd HH:mm:ss") + } + } + + def findResultFromDB(city, datas){ + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + def total = datas.size() + for(def i=0;i