Merge remote-tracking branch 'origin/master'
This commit is contained in:
+214
@@ -0,0 +1,214 @@
|
||||
package com.fangjia.script.gj.data_handle
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import java.util.concurrent.CopyOnWriteArrayList
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 第三方小区与我们小区映射关系表
|
||||
*/
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ThirdPriceMappingBatch extends BaseBatch{
|
||||
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
|
||||
final DBCollection thirdPartyPriceColl = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018)
|
||||
|
||||
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||
|
||||
|
||||
final DBCollection dataProcessColl = MongoUtils.getCollectionByName("fangjia_craw", "data_process", "192.168.0.235", 27018)
|
||||
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ThirdPriceMappingBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
// //将新增数据同步过来
|
||||
// addNemCommunity()
|
||||
// //接口进行匹配
|
||||
match()
|
||||
|
||||
// addOldMatchResult()
|
||||
}
|
||||
|
||||
def match(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
for(def c : citys){
|
||||
def query = new BasicDBObject("city", c).append("match_id", new BasicDBObject('$exists', false))
|
||||
def noMatchDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||
matchByCity(c, noMatchDatas)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def matchByCity(def c, def noMatchDatas){
|
||||
def totalCount = new AtomicLong()
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(noMatchDatas.size())
|
||||
def total = noMatchDatas.size()
|
||||
for(def i=0;i<noMatchDatas.size();i++){
|
||||
final def nmd = noMatchDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def address = nmd.city
|
||||
if(nmd.district){
|
||||
address = address + nmd.district
|
||||
}
|
||||
address = address + nmd.name
|
||||
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "property", true)
|
||||
if(jsonObject) {
|
||||
def udata = new BasicDBObject()
|
||||
udata.put("match_id",jsonObject.get("id"))
|
||||
udata.put("match_city",jsonObject.get("city"))
|
||||
udata.put("match_district",jsonObject.get("district"))
|
||||
udata.put("match_name",jsonObject.get("name"))
|
||||
udata.put("match_type",jsonObject.get("propertyType"))
|
||||
udata.put("match_credit",jsonObject.get("credit"))
|
||||
thirdSeaweedMappingColl.update(new BasicDBObject("_id", nmd._id), new BasicDBObject('$set', udata))
|
||||
|
||||
println addCount.get()+"/"+total +" [success] "+ udata.get("match_district")+" "+udata.get("match_name")
|
||||
}else{
|
||||
println addCount.get()+"/"+total +" [fail] "+address
|
||||
}
|
||||
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return addCount.get();
|
||||
}
|
||||
|
||||
def addNemCommunity(){
|
||||
def citys = CommonUtils.findAllCity()
|
||||
def total = 0
|
||||
for(def c : citys){
|
||||
def crawlDatas = thirdPartyPriceColl.find(new BasicDBObject("city", c).append("s_date", curDate)).toList()
|
||||
def addCount = addDatasToDB(crawlDatas)
|
||||
println c + " 新增:"+ addCount
|
||||
total = total + addCount
|
||||
}
|
||||
println total +" 新增总计:"+total
|
||||
|
||||
}
|
||||
|
||||
|
||||
def addDatasToDB(def crawlDatas){
|
||||
def totalCount = new AtomicLong()
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(crawlDatas.size())
|
||||
for(def i=0;i<crawlDatas.size();i++){
|
||||
final def sd = crawlDatas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
sd.put("district", sd.region)
|
||||
sd.removeField("region")
|
||||
// sd.removeField("url")
|
||||
sd.removeField("s_date")
|
||||
totalCount.addAndGet(1)
|
||||
def query = new BasicDBObject("url", sd.url)
|
||||
def dbo = thirdSeaweedMappingColl.findOne(query)
|
||||
if(dbo){
|
||||
def udata = new BasicDBObject()
|
||||
sd.each {k,v->
|
||||
udata.put(k,v)
|
||||
}
|
||||
udata.put("m_date", new Date())
|
||||
thirdSeaweedMappingColl.update(query, new BasicDBObject('$set', udata))
|
||||
}else{
|
||||
sd.put("m_date", new Date())
|
||||
thirdSeaweedMappingColl.save(sd)
|
||||
addCount.addAndGet(1)
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
return addCount.get();
|
||||
}
|
||||
|
||||
def addOldMatchResult(){
|
||||
def citys = ["上海"]//CommonUtils.findAllCity()
|
||||
def keys = new BasicDBObject()
|
||||
keys.put("_id", 1)
|
||||
|
||||
for(def c : citys){
|
||||
def datas = thirdSeaweedMappingColl.find(new BasicDBObject("city", c).append("old_match_id", new BasicDBObject('$exists', false)), keys).toList()
|
||||
println c +" 待关联数据:"+datas.size()
|
||||
findResultFromDB(c, datas)
|
||||
println c + " 关联数据结束 "+DateUtils.formatDateByPattern(new Date(),"yyyy-MM-dd HH:mm:ss")
|
||||
}
|
||||
}
|
||||
|
||||
def findResultFromDB(city, datas){
|
||||
def addCount = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||
def total = datas.size()
|
||||
for(def i=0;i<datas.size();i++){
|
||||
final def d = datas[i]
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try{
|
||||
addCount.addAndGet(1)
|
||||
def query = new BasicDBObject().append("s_date", curDate).append("city", city).append("tid", d._id.toString())
|
||||
def obj = dataProcessColl.findOne(query)
|
||||
if(obj) {
|
||||
def dist = CommonDatautils.findDistrict(obj.match_city, obj.match_region, obj.match_name, "district")
|
||||
def udata = new BasicDBObject()
|
||||
if(dist){
|
||||
udata.put("old_match_id",dist._id.toString())
|
||||
}
|
||||
udata.put("old_match_city",obj.match_city)
|
||||
udata.put("old_match_district",obj.match_region)
|
||||
udata.put("old_match_name",obj.match_name)
|
||||
udata.put("old_match_way",obj.type)
|
||||
thirdSeaweedMappingColl.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||
|
||||
println addCount.get()+"/"+total +" [旧匹配找到] "
|
||||
}else{
|
||||
println addCount.get()+"/"+total +" [旧匹配没找到] "
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -249,9 +249,9 @@ class FangjiaClientUtils {
|
||||
def url = null;
|
||||
try {
|
||||
address = address.replace("{","").replace("\\","")
|
||||
// url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
url = "http://192.168.0.93:8080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.20.242:5001/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
// url = "http://192.168.0.19:31080/FangJiaNlpService/match?address="+URLEncoder.encode(address,"UTF-8")+"&useMap=${useMap}"
|
||||
if(city){
|
||||
url = url + "&city="+URLEncoder.encode(city,"UTF-8")
|
||||
}
|
||||
|
||||
+4
@@ -70,6 +70,10 @@ class SynchDistrictPrice extends BaseBatch{
|
||||
data.put("estate_type",d.property_type)
|
||||
data.put("s_date", SHORT_DATE)
|
||||
data.put("url", d.url)
|
||||
data.put("id",d.id)
|
||||
data.put("ownership_type",d.ownership_type)
|
||||
data.put("property_type",d.property_type)
|
||||
data.put("build_type",d.build_type)
|
||||
data.put("c_date", new Date())
|
||||
data.put("m_date", new Date())
|
||||
rets << data
|
||||
|
||||
Reference in New Issue
Block a user