第三方均价匹配,用新匹配

This commit is contained in:
2024-09-24 17:49:42 +08:00
parent d983a79927
commit 5dc4224847
@@ -0,0 +1,150 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 第三方小区与我们小区映射关系表
*/
@BatchOwner(Watcher.GaoJuan)
class ThirdPriceMappingBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new ThirdPriceMappingBatch(), args);
}
@Override
public void process(Object args) {
//将新增数据同步过来
addNemCommunity()
//接口进行匹配
match()
}
def match(){
def citys = CommonUtils.findAllCity()
for(def c : citys){
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
matchByCity(c, noMatchDatas)
}
}
def matchByCity(def c, def noMatchDatas){
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(noMatchDatas.size())
def total = noMatchDatas.size()
for(def i=0;i<noMatchDatas.size();i++){
final def nmd = noMatchDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def address = nmd.city + nmd.district + nmd.name
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true)
if(jsonObject) {
def udata = new BasicDBObject()
udata.put("match_id",jsonObject.get("id"))
udata.put("match_city",jsonObject.get("city"))
udata.put("match_district",jsonObject.get("district"))
udata.put("match_name",jsonObject.get("name"))
udata.put("match_type",jsonObject.get("propertyType"))
udata.put("match_credit",jsonObject.get("credit"))
thirdSeaweedMappingColl.update(new BasicDBObject("_id", nmd._id), new BasicDBObject('$set', udata))
println addCount.get()+"/"+total +" [success] "+ udata.get("match_district")+" "+udata.get("match_name")
}else{
println addCount.get()+"/"+total +" [fail] "+address
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return addCount.get();
}
def addNemCommunity(){
def citys = CommonUtils.findAllCity()
def total = 0
for(def c : citys){
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
def addCount = addDatasToDB(crawlDatas)
println c + " 新增:"+ addCount
total = total + addCount
}
println total +" 新增总计:"+total
}
def addDatasToDB(def crawlDatas){
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(crawlDatas.size())
for(def i=0;i<crawlDatas.size();i++){
final def sd = crawlDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
sd.put("district", sd.region)
sd.removeField("region")
sd.removeField("url")
sd.removeField("s_date")
totalCount.addAndGet(1)
def query = new BasicDBObject("url", sd.url)
def dbo = thirdSeaweedMappingColl.findOne(query)
if(dbo){
def udata = new BasicDBObject()
sd.each {k,v->
udata.put(k,v)
}
udata.put("m_date", new Date())
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata))
}else{
sd.put("m_date", new Date())
thirdSeaweedMappingColl.save(sd)
addCount.addAndGet(1)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return addCount.get();
}
}