Merge remote-tracking branch 'origin/master'

This commit is contained in:
majintao
2024-09-25 16:02:15 +08:00
3 changed files with 220 additions and 2 deletions
@@ -0,0 +1,214 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
/**
* 第三方小区与我们小区映射关系表
*/
@BatchOwner(Watcher.GaoJuan)
class ThirdPriceMappingBatch extends BaseBatch{
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
static main(args) {
runBatch(new ThirdPriceMappingBatch(), args);
}
@Override
public void process(Object args) {
// //将新增数据同步过来
// addNemCommunity()
// //接口进行匹配
match()
// addOldMatchResult()
}
def match(){
def citys = CommonUtils.findAllCity()
for(def c : citys){
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
matchByCity(c, noMatchDatas)
}
}
def matchByCity(def c, def noMatchDatas){
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(noMatchDatas.size())
def total = noMatchDatas.size()
for(def i=0;i<noMatchDatas.size();i++){
final def nmd = noMatchDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def address = nmd.city
if(nmd.district){
address = address + nmd.district
}
address = address + nmd.name
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true)
if(jsonObject) {
def udata = new BasicDBObject()
udata.put("match_id",jsonObject.get("id"))
udata.put("match_city",jsonObject.get("city"))
udata.put("match_district",jsonObject.get("district"))
udata.put("match_name",jsonObject.get("name"))
udata.put("match_type",jsonObject.get("propertyType"))
udata.put("match_credit",jsonObject.get("credit"))
thirdSeaweedMappingColl.update(new BasicDBObject("_id", nmd._id), new BasicDBObject('$set', udata))
println addCount.get()+"/"+total +" [success] "+ udata.get("match_district")+" "+udata.get("match_name")
}else{
println addCount.get()+"/"+total +" [fail] "+address
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return addCount.get();
}
def addNemCommunity(){
def citys = CommonUtils.findAllCity()
def total = 0
for(def c : citys){
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
def addCount = addDatasToDB(crawlDatas)
println c + " 新增:"+ addCount
total = total + addCount
}
println total +" 新增总计:"+total
}
def addDatasToDB(def crawlDatas){
def totalCount = new AtomicLong()
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(crawlDatas.size())
for(def i=0;i<crawlDatas.size();i++){
final def sd = crawlDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
sd.put("district", sd.region)
sd.removeField("region")
// sd.removeField("url")
sd.removeField("s_date")
totalCount.addAndGet(1)
def query = new BasicDBObject("url", sd.url)
def dbo = thirdSeaweedMappingColl.findOne(query)
if(dbo){
def udata = new BasicDBObject()
sd.each {k,v->
udata.put(k,v)
}
udata.put("m_date", new Date())
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata))
}else{
sd.put("m_date", new Date())
thirdSeaweedMappingColl.save(sd)
addCount.addAndGet(1)
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
return addCount.get();
}
def addOldMatchResult(){
def citys = ["上海"]//CommonUtils.findAllCity()
def keys = new BasicDBObject()
keys.put("_id", 1)
for(def c : citys){
def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList()
println c +" 待关联数据:"+datas.size()
findResultFromDB(c, datas)
println c + " 关联数据结束 "+DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd HH:mm:ss")
}
}
def findResultFromDB(city, datas){
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
def total = datas.size()
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
addCount.addAndGet(1)
def query = new BasicDBObject().append("s_date", curDate).append("city", city).append("tid", d._id.toString())
def obj = dataProcessColl.findOne(query)
if(obj) {
def dist = CommonDatautils.findDistrict(obj.match_city, obj.match_region, obj.match_name, "district")
def udata = new BasicDBObject()
if(dist){
udata.put("old_match_id",dist._id.toString())
}
udata.put("old_match_city",obj.match_city)
udata.put("old_match_district",obj.match_region)
udata.put("old_match_name",obj.match_name)
udata.put("old_match_way",obj.type)
thirdSeaweedMappingColl.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
println addCount.get()+"/"+total +" [旧匹配找到] "
}else{
println addCount.get()+"/"+total +" [旧匹配没找到] "
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
}
@@ -249,9 +249,9 @@ class FangjiaClientUtils {
def url = null;
try {
address = address.replace("{","").replace("\\","")
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
if(city){
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
}
@@ -70,6 +70,10 @@ class SynchDistrictPrice extends BaseBatch{
data.put("estate_type",d.property_type)
data.put("s_date", SHORT_DATE)
data.put("url", d.url)
data.put("id",d.id)
data.put("ownership_type",d.ownership_type)
data.put("property_type",d.property_type)
data.put("build_type",d.build_type)
data.put("c_date", new Date())
data.put("m_date", new Date())
rets << data