Merge branch 'script_gj' into 'master'
小区字段自动填充优化 See merge request !1073
This commit is contained in:
+22
@@ -103,6 +103,8 @@ class CommonDatautils {
|
|||||||
|
|
||||||
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
public static DSCollection getHouseDataStoreConnection(){
|
public static DSCollection getHouseDataStoreConnection(){
|
||||||
if(house_dataStoreConnection != null) {
|
if(house_dataStoreConnection != null) {
|
||||||
return house_dataStoreConnection
|
return house_dataStoreConnection
|
||||||
@@ -4967,6 +4969,20 @@ class CommonDatautils {
|
|||||||
return regionMap
|
return regionMap
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def queryRangeByCat(cat){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("type",cat)
|
||||||
|
q.put("shape",new BasicDBObject('$exists', true))
|
||||||
|
def keys = new BasicDBObject()
|
||||||
|
keys.put("city_short",1)
|
||||||
|
keys.put("district_short",1)
|
||||||
|
keys.put("city",1)
|
||||||
|
keys.put("district",1)
|
||||||
|
keys.put("mzb_district_name", 1)
|
||||||
|
keys.put("shape", 1)
|
||||||
|
return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList()
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
static def queryAllRegionNameByCity(){
|
static def queryAllRegionNameByCity(){
|
||||||
|
|
||||||
@@ -5522,4 +5538,10 @@ class CommonDatautils {
|
|||||||
return 0
|
return 0
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static def findBusinessByCity(city){
|
||||||
|
def q = new BasicDBObject()
|
||||||
|
q.put("short_city", city)
|
||||||
|
return bk_business_coll.find(q).toList()
|
||||||
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
+268
@@ -0,0 +1,268 @@
|
|||||||
|
package com.fangjia.script.gj.data_handle
|
||||||
|
|
||||||
|
import com.fangjia.core.RegionData
|
||||||
|
import com.fangjia.script.commData.CommonDatautils
|
||||||
|
import com.fangjia.script.common.DataHandleUtils
|
||||||
|
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||||
|
import com.fangjia.script.gj.xiamen.ElasticUtils89
|
||||||
|
import com.fangjia.script.lixue.util.DBUtils
|
||||||
|
import com.fangjia.script.looplineinfo.InsidePolygon
|
||||||
|
import com.fangjia.script.util.DbUtils
|
||||||
|
import com.fangjia.script.util.ElasticUtils
|
||||||
|
import com.fangjia.script.util.MongoUtils
|
||||||
|
import com.mongodb.BasicDBObject
|
||||||
|
import com.mongodb.DBCollection
|
||||||
|
import org.bson.types.ObjectId
|
||||||
|
import org.elasticsearch.action.bulk.BulkRequest
|
||||||
|
import org.elasticsearch.action.bulk.BulkResponse
|
||||||
|
import org.elasticsearch.action.index.IndexRequest
|
||||||
|
import org.elasticsearch.action.support.WriteRequest
|
||||||
|
import org.elasticsearch.action.update.UpdateRequest
|
||||||
|
import org.elasticsearch.client.RequestOptions
|
||||||
|
import org.elasticsearch.common.xcontent.XContentType
|
||||||
|
|
||||||
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
|
import java.util.concurrent.CountDownLatch
|
||||||
|
import java.util.concurrent.ExecutorService
|
||||||
|
import java.util.concurrent.Executors
|
||||||
|
import java.util.concurrent.atomic.AtomicLong
|
||||||
|
|
||||||
|
class BusinessQuarterProcess {
|
||||||
|
|
||||||
|
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
|
||||||
|
def fullCityMap = CommonDatautils.queryFullNameCity()
|
||||||
|
DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020)
|
||||||
|
|
||||||
|
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
|
||||||
|
|
||||||
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
|
|
||||||
|
static main(args) {
|
||||||
|
def task = new BusinessQuarterProcess()
|
||||||
|
task.run()
|
||||||
|
}
|
||||||
|
def run(){
|
||||||
|
// synchBusinessData()
|
||||||
|
// updateBusinessQuarter()
|
||||||
|
// addRegion()
|
||||||
|
synchBusinessToEs()
|
||||||
|
}
|
||||||
|
|
||||||
|
def addRegion(){
|
||||||
|
def list = CommonDatautils.queryRangeByCat("区域")
|
||||||
|
println list.size()
|
||||||
|
def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2")))
|
||||||
|
for(def d : dbcursor){
|
||||||
|
if(d.district){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
for(def regionShape : list){
|
||||||
|
if(d.short_city!=regionShape.city_short){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates))
|
||||||
|
if(flag){
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("district", regionShape.mzb_district_name)
|
||||||
|
udata.put("short_district", regionShape.district_short)
|
||||||
|
bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def synchBusinessData(){
|
||||||
|
def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList()
|
||||||
|
synchMongoProcess(datas)
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
def synchMongoProcess(datas){
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(datas.size())
|
||||||
|
for(def i=0;i<datas.size();i++){
|
||||||
|
final def d = datas[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
createData(d)
|
||||||
|
println d.city +" "+totalCount.get() +"/"+datas.size()
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
println d.city +" "+d.name
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
}
|
||||||
|
|
||||||
|
def createData(d){
|
||||||
|
def row = new BasicDBObject()
|
||||||
|
row.put("_id", d._id)
|
||||||
|
row.put("short_province", d.province)
|
||||||
|
def short_city = RegionData.formatCity(d.city)
|
||||||
|
row.put("short_city", short_city)
|
||||||
|
row.put("name", d.name)
|
||||||
|
row.put("bd_location", ["lon":DataHandleUtils.getDoubleRound(6,d.longitude), "lat":DataHandleUtils.getDoubleRound(6,d.latitude)])
|
||||||
|
|
||||||
|
def data = fullCityMap.get(short_city)
|
||||||
|
if(data){
|
||||||
|
row.put("province", data.province)
|
||||||
|
row.put("city", data.city)
|
||||||
|
}else{
|
||||||
|
println d.city + " 没找到简称"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def gdPoints = FangjiaClientUtils.convert(d.longitude+","+d.latitude, 3)
|
||||||
|
if(gdPoints){
|
||||||
|
row.put("location", ["lon":DataHandleUtils.getDoubleRound(6,gdPoints[0]), "lat":DataHandleUtils.getDoubleRound(6,gdPoints[1])])
|
||||||
|
}
|
||||||
|
def border = d.border
|
||||||
|
def bd_shapes = []
|
||||||
|
def gd_shapes = []
|
||||||
|
if(border){
|
||||||
|
def arrs = border.split(";")
|
||||||
|
if(border.contains("|")){
|
||||||
|
println d._id.toString()+" 可能含多个围栏"
|
||||||
|
}
|
||||||
|
|
||||||
|
for(def i=0;i<arrs.size();i++){
|
||||||
|
def str = arrs[i]
|
||||||
|
def arr = str.split(",")
|
||||||
|
def bd_lng = DataHandleUtils.getDoubleRound(6, arr[0].toString().toDouble())
|
||||||
|
def bd_lat = DataHandleUtils.getDoubleRound(6, arr[1].toString().toDouble())
|
||||||
|
if(!bd_shapes.contains([bd_lng, bd_lat]) ){
|
||||||
|
bd_shapes << [bd_lng, bd_lat]
|
||||||
|
}
|
||||||
|
if(bd_shapes.contains([bd_lng, bd_lat]) && i+1==arrs.size() && bd_shapes[0]==[bd_lng, bd_lat]){
|
||||||
|
bd_shapes << [bd_lng, bd_lat]
|
||||||
|
}
|
||||||
|
def tempGdPoints = FangjiaClientUtils.convert(bd_lng+","+bd_lat, 3)
|
||||||
|
if(tempGdPoints){
|
||||||
|
def gdlist = [DataHandleUtils.getDoubleRound(6,tempGdPoints[0]), DataHandleUtils.getDoubleRound(6,tempGdPoints[1])]
|
||||||
|
if(!gd_shapes.contains(gdlist)) {
|
||||||
|
gd_shapes << gdlist
|
||||||
|
}
|
||||||
|
if(gd_shapes.contains(gdlist) && i+1==arrs.size() && gd_shapes[0]==gdlist){
|
||||||
|
gd_shapes << gdlist
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if(bd_shapes[0] != bd_shapes[bd_shapes.size()-1]){
|
||||||
|
bd_shapes << bd_shapes[0]
|
||||||
|
gd_shapes << gd_shapes[0]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
def gdMap = [:]
|
||||||
|
gdMap.put("type" , "polygon")
|
||||||
|
gdMap.put("coordinates", [gd_shapes])
|
||||||
|
row.put("shape", gdMap)
|
||||||
|
|
||||||
|
def bdMap = [:]
|
||||||
|
bdMap.put("type" , "polygon")
|
||||||
|
bdMap.put("coordinates", [bd_shapes])
|
||||||
|
row.put("bd_shape", bdMap)
|
||||||
|
row.put("c_date", new Date())
|
||||||
|
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
|
||||||
|
}
|
||||||
|
def updateBusinessQuarter(){
|
||||||
|
def citys = CommonDatautils.findAllCity()
|
||||||
|
for(def c : citys){
|
||||||
|
if(["上海","北京"].contains(c)){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
|
||||||
|
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
|
||||||
|
businessProcess(c, districts, cityBusinessList)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def businessProcess(c,districts, cityBusinessList){
|
||||||
|
def updateIndex = 0
|
||||||
|
for(def d : districts){
|
||||||
|
if(!d.business_quarter){
|
||||||
|
def businessQuarter = findBusinessByRange(d, cityBusinessList)
|
||||||
|
if(businessQuarter){
|
||||||
|
updateIndex ++
|
||||||
|
def udata = new BasicDBObject()
|
||||||
|
udata.put("business_quarter", businessQuarter)
|
||||||
|
udata.put("m_date", new Date())
|
||||||
|
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
println c +" "+ districts.size() +" "+updateIndex
|
||||||
|
}
|
||||||
|
def findBusinessByRange(d, cityBusinessList){
|
||||||
|
def name= null
|
||||||
|
for(def shapeObj : cityBusinessList){
|
||||||
|
if(d.lng){
|
||||||
|
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(shapeObj.shape.coordinates))
|
||||||
|
if(flag){
|
||||||
|
name = shapeObj.name
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return name
|
||||||
|
}
|
||||||
|
|
||||||
|
def synchBusinessToEs(){
|
||||||
|
def dbcursor = bk_business_coll.find(new BasicDBObject())
|
||||||
|
def rows = []
|
||||||
|
def index = 0
|
||||||
|
for(def d : dbcursor){
|
||||||
|
index++
|
||||||
|
if(index<5000){
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// rows << d
|
||||||
|
// if(rows.size()>=100){
|
||||||
|
// updateByEs(rows)
|
||||||
|
// rows.clear()
|
||||||
|
// }
|
||||||
|
updateByEs(d, index)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
def updateByEs(row, index){
|
||||||
|
BulkRequest request = new BulkRequest();
|
||||||
|
def client = ElasticUtils89.getClient("192.168.0.89")
|
||||||
|
int processedCount = 0;
|
||||||
|
|
||||||
|
def id = row.get("_id").toString()
|
||||||
|
def doc = [:]
|
||||||
|
doc.put("province",row.province)
|
||||||
|
doc.put("short_province",row.short_province)
|
||||||
|
doc.put("city",row.city)
|
||||||
|
doc.put("short_city",row.short_city)
|
||||||
|
doc.put("district",row.district)
|
||||||
|
doc.put("short_district",row.short_district)
|
||||||
|
doc.put("name", row.name)
|
||||||
|
doc.put("location", row.location)
|
||||||
|
doc.put("bd_location", row.bd_location)
|
||||||
|
doc.put("shape", row.shape)
|
||||||
|
doc.put("bd_shape", row.bd_shape)
|
||||||
|
|
||||||
|
|
||||||
|
request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON));
|
||||||
|
if (request.numberOfActions() >= 0) {
|
||||||
|
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
|
||||||
|
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
|
||||||
|
if (bulkResponse.hasFailures()) {
|
||||||
|
// 处理错误
|
||||||
|
println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage()
|
||||||
|
} else {
|
||||||
|
processedCount += request.numberOfActions();
|
||||||
|
// println "Processed " + processedCount + " documents"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
}
|
||||||
+89
-14
@@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject
|
|||||||
import com.mongodb.DBCollection
|
import com.mongodb.DBCollection
|
||||||
import com.rabbitmq.client.impl.AMQImpl
|
import com.rabbitmq.client.impl.AMQImpl
|
||||||
import org.bson.types.ObjectId
|
import org.bson.types.ObjectId
|
||||||
|
import org.elasticsearch.index.query.QueryBuilder
|
||||||
|
import org.elasticsearch.index.query.QueryBuilders
|
||||||
|
|
||||||
import java.text.SimpleDateFormat
|
import java.text.SimpleDateFormat
|
||||||
import java.util.concurrent.CopyOnWriteArrayList
|
import java.util.concurrent.CopyOnWriteArrayList
|
||||||
@@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
|
||||||
|
|
||||||
//
|
//
|
||||||
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
|
def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房",
|
||||||
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
|
||||||
|
|
||||||
|
|
||||||
@@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
|
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
|
||||||
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
|
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
|
||||||
|
|
||||||
|
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
|
||||||
|
|
||||||
final static def EMAILS = [
|
final static def EMAILS = [
|
||||||
"gaojuan@fangjia.com"
|
"gaojuan@fangjia.com"
|
||||||
@@ -115,18 +118,12 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
headers << fieldsNameMap.get(it)
|
headers << fieldsNameMap.get(it)
|
||||||
}
|
}
|
||||||
|
|
||||||
headers << "抓取量"
|
|
||||||
headers << "匹配量"
|
|
||||||
headers << "0.9分以上量"
|
|
||||||
headers << "匹配率"
|
|
||||||
headers << "0.9以上匹配率"
|
|
||||||
// analysRets << headers
|
|
||||||
|
|
||||||
for(def c : citys){
|
for(def c : citys){
|
||||||
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
|
||||||
if(totalCount){
|
if(totalCount){
|
||||||
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
|
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
|
||||||
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
|
||||||
|
analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
println analysRets.toString()
|
println analysRets.toString()
|
||||||
@@ -139,8 +136,10 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
|
|
||||||
sourceMap.each { seaweedmapSource, esWebSource ->
|
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||||
def esRets = findEsDatas(c, esWebSource)
|
def esRets = findEsDatas(c, esWebSource)
|
||||||
|
|
||||||
def matchRets1 = matchApi(esRets)
|
def matchRets1 = matchApi(esRets)
|
||||||
def matchRets2 = selectRestByCredit(matchRets1)
|
def matchRets2 = selectRestByCredit(matchRets1)
|
||||||
|
|
||||||
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
|
||||||
|
|
||||||
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
|
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
|
||||||
@@ -172,6 +171,65 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def addFieldFromMatchMapping(c, totalCount, analysRets){
|
||||||
|
def fields = fieldsMap.keySet() as List
|
||||||
|
|
||||||
|
sourceMap.each { seaweedmapSource, esWebSource ->
|
||||||
|
def esRets = findMatchSameDatas(c, esWebSource)
|
||||||
|
println esWebSource +" 新老算法匹配相同量:"+esRets.size()
|
||||||
|
|
||||||
|
def updateRets = processFieldsFromMatch(c , esRets, esWebSource)
|
||||||
|
|
||||||
|
def precentMap = fieldUpdate(totalCount, updateRets)
|
||||||
|
println c +" "+precentMap.toString()
|
||||||
|
updateDB(c, updateRets)
|
||||||
|
println "------------------------------------------------------------------------------"
|
||||||
|
|
||||||
|
def analyRet = []
|
||||||
|
analyRet << seaweedmapSource
|
||||||
|
analyRet << c
|
||||||
|
fields.each{key->
|
||||||
|
analyRet << precentMap.get(key)
|
||||||
|
}
|
||||||
|
analyRet << esRets.size()
|
||||||
|
analysRets << analyRet
|
||||||
|
}
|
||||||
|
return analysRets;
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def findMatchSameDatas(c, esWebSource){
|
||||||
|
def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true))
|
||||||
|
def siteDatas = thirdSeaweedMappingColl.find(query).toList()
|
||||||
|
def rows = new CopyOnWriteArrayList()
|
||||||
|
def totalCount = new AtomicLong()
|
||||||
|
final CountDownLatch latch = new CountDownLatch(siteDatas.size())
|
||||||
|
for(def i=0;i<siteDatas.size();i++){
|
||||||
|
final def d = siteDatas[i]
|
||||||
|
fixedThreadPool.execute(new Runnable() {
|
||||||
|
void run() {
|
||||||
|
try{
|
||||||
|
totalCount.addAndGet(1)
|
||||||
|
def obj = findCommunities(d.url)
|
||||||
|
if(obj){
|
||||||
|
obj.match_id = d.match_id
|
||||||
|
rows << obj
|
||||||
|
}
|
||||||
|
// println d.city +" "+totalCount.get() +"/"+siteDatas.size()
|
||||||
|
} catch (Exception e) {
|
||||||
|
e.printStackTrace()
|
||||||
|
println d.city +" "+d.name
|
||||||
|
} finally {
|
||||||
|
latch.countDown()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await()
|
||||||
|
return rows
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
def addFieldFromRonghe(c, totalCount, rh_analysRets){
|
def addFieldFromRonghe(c, totalCount, rh_analysRets){
|
||||||
@@ -241,14 +299,10 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
fields << "city"
|
fields << "city"
|
||||||
fields << "estate_type2"
|
fields << "estate_type2"
|
||||||
def fieldValueMap = [:]
|
def fieldValueMap = [:]
|
||||||
def rets = []
|
|
||||||
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
|
|
||||||
for(def matchDist : matchRets){
|
for(def matchDist : matchRets){
|
||||||
|
|
||||||
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
|
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
|
||||||
// continue
|
// continue
|
||||||
// }
|
// }
|
||||||
|
|
||||||
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
|
||||||
if(!dist){
|
if(!dist){
|
||||||
println c+" "+matchDist.match_id+" 不存在"
|
println c+" "+matchDist.match_id+" 不存在"
|
||||||
@@ -256,7 +310,6 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
def newValues = strandField(c, dist, matchDist)
|
def newValues = strandField(c, dist, matchDist)
|
||||||
if(newValues){
|
if(newValues){
|
||||||
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
|
|
||||||
println dist._id.toString()+" "+newValues.toString()
|
println dist._id.toString()+" "+newValues.toString()
|
||||||
fieldValueMap.put(dist._id.toString(), newValues)
|
fieldValueMap.put(dist._id.toString(), newValues)
|
||||||
}
|
}
|
||||||
@@ -662,4 +715,26 @@ class DistrictFieldAddBatch extends BaseBatch{
|
|||||||
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
|
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def findCommunities(url){
|
||||||
|
if(!url){
|
||||||
|
return null
|
||||||
|
}
|
||||||
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
|
||||||
|
def rets = []
|
||||||
|
def datas = null
|
||||||
|
try{
|
||||||
|
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
|
||||||
|
}catch(Exception e){
|
||||||
|
findCommunities(url)
|
||||||
|
}
|
||||||
|
def obj = null
|
||||||
|
if(datas) {
|
||||||
|
for (int k = 0; k < datas.size(); k++) {
|
||||||
|
def d = datas[k];
|
||||||
|
obj = d
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return obj;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+3
-3
@@ -52,8 +52,8 @@ class AddDistImagesBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
def findAllEsData2(city, mdate){
|
def findAllEsData2(city, mdate){
|
||||||
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
|
||||||
|
//安居客小程序
|
||||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
|
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "贝壳找房"))
|
||||||
.must(date_range).must(date_range)
|
.must(date_range).must(date_range)
|
||||||
|
|
||||||
def rets = []
|
def rets = []
|
||||||
@@ -110,7 +110,7 @@ class AddDistImagesBatch extends BaseBatch{
|
|||||||
try{
|
try{
|
||||||
addCount.addAndGet(1)
|
addCount.addAndGet(1)
|
||||||
|
|
||||||
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
|
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","贝壳找房").append("match_id", new BasicDBObject('$exists', true))
|
||||||
def obj = thirdSeaweedMappingColl.findOne(query)
|
def obj = thirdSeaweedMappingColl.findOne(query)
|
||||||
if(obj) {
|
if(obj) {
|
||||||
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
|
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))
|
||||||
|
|||||||
Reference in New Issue
Block a user