匹配19换93
This commit is contained in:
+68
-4
@@ -3,12 +3,15 @@ package com.fangjia.script.gj.data_handle
|
|||||||
import com.fangjia.script.BaseBatch
|
import com.fangjia.script.BaseBatch
|
||||||
import com.fangjia.script.BatchOwner
|
import com.fangjia.script.BatchOwner
|
||||||
import com.fangjia.script.Watcher
|
import com.fangjia.script.Watcher
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
import com.fangjia.script.util.CommonUtils
|
import com.fangjia.script.util.CommonUtils
|
||||||
import com.fangjia.script.util.DateUtils
|
import com.fangjia.script.util.DateUtils
|
||||||
import com.fangjia.script.util.MongoUtils
|
import com.fangjia.script.util.MongoUtils
|
||||||
import com.mongodb.BasicDBObject
|
import com.mongodb.BasicDBObject
|
||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
import java.util.concurrent.CountDownLatch
|
import java.util.concurrent.CountDownLatch
|
||||||
@@ -28,6 +31,9 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
|
|
||||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||||
|
|
||||||
|
|
||||||
|
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
|
||||||
|
|
||||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||||
|
|
||||||
static main(args) {
|
static main(args) {
|
||||||
@@ -36,10 +42,12 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
public void process(Object args) {
|
public void process(Object args) {
|
||||||
//将新增数据同步过来
|
// //将新增数据同步过来
|
||||||
addNemCommunity()
|
// addNemCommunity()
|
||||||
//接口进行匹配
|
// //接口进行匹配
|
||||||
match()
|
match()
|
||||||
|
|
||||||
|
// addOldMatchResult()
|
||||||
}
|
}
|
||||||
|
|
||||||
def match(){
|
def match(){
|
||||||
@@ -63,7 +71,11 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
void run() {
|
void run() {
|
||||||
try{
|
try{
|
||||||
addCount.addAndGet(1)
|
addCount.addAndGet(1)
|
||||||
def address = nmd.city + nmd.district + nmd.name
|
def address = nmd.city
|
||||||
|
if(nmd.district){
|
||||||
|
address = address + nmd.district
|
||||||
|
}
|
||||||
|
address = address + nmd.name
|
||||||
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true)
|
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true)
|
||||||
if(jsonObject) {
|
if(jsonObject) {
|
||||||
def udata = new BasicDBObject()
|
def udata = new BasicDBObject()
|
||||||
@@ -147,4 +159,56 @@ class ThirdPriceMappingBatch extends BaseBatch{
|
|||||||
latch.await()
|
latch.await()
|
||||||
return addCount.get();
|
return addCount.get();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def addOldMatchResult(){
|
||||||
|
def citys = ["上海"]//CommonUtils.findAllCity()
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
keys.put("_id", 1)
|
||||||
|
|
||||||
|
for(def c : citys){
|
||||||
|
def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList()
|
||||||
|
println c +" 待关联数据:"+datas.size()
|
||||||
|
findResultFromDB(c, datas)
|
||||||
|
println c + " 关联数据结束 "+DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd HH:mm:ss")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def findResultFromDB(city, datas){
|
||||||
|
def addCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||||
|
def total = datas.size()
|
||||||
|
for(def i=0;i<datas.size();i++){
|
||||||
|
final def d = datas[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
addCount.addAndGet(1)
|
||||||
|
def query = new BasicDBObject().append("s_date", curDate).append("city", city).append("tid", d._id.toString())
|
||||||
|
def obj = dataProcessColl.findOne(query)
|
||||||
|
if(obj) {
|
||||||
|
def dist = CommonDatautils.findDistrict(obj.match_city, obj.match_region, obj.match_name, "district")
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
if(dist){
|
||||||
|
udata.put("old_match_id",dist._id.toString())
|
||||||
|
}
|
||||||
|
udata.put("old_match_city",obj.match_city)
|
||||||
|
udata.put("old_match_district",obj.match_region)
|
||||||
|
udata.put("old_match_name",obj.match_name)
|
||||||
|
udata.put("old_match_way",obj.type)
|
||||||
|
thirdSeaweedMappingColl.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||||
|
|
||||||
|
println addCount.get()+"/"+total +" [旧匹配找到] "
|
||||||
|
}else{
|
||||||
|
println addCount.get()+"/"+total +" [旧匹配没找到] "
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+2
-2
@@ -249,9 +249,9 @@ class FangjiaClientUtils {
|
|||||||
def url = null;
|
def url = null;
|
||||||
try {
|
try {
|
||||||
address = address.replace("{","").replace("\\","")
|
address = address.replace("{","").replace("\\","")
|
||||||
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||||
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||||
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||||
if(city){
|
if(city){
|
||||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user