小区字段自动填充优化

This commit is contained in:
2025-02-11 16:12:02 +08:00
parent c717f3cf6c
commit cf3fbdcc80
4 changed files with 382 additions and 17 deletions
@@ -103,6 +103,8 @@ class CommonDatautils {
public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com")
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static DSCollection getHouseDataStoreConnection(){
if(house_dataStoreConnection != null) {
return house_dataStoreConnection
@@ -4967,6 +4969,20 @@ class CommonDatautils {
return regionMap
}
static def queryRangeByCat(cat){
def q = new BasicDBObject()
q.put("type",cat)
q.put("shape",new BasicDBObject('$exists', true))
def keys = new BasicDBObject()
keys.put("city_short",1)
keys.put("district_short",1)
keys.put("city",1)
keys.put("district",1)
keys.put("mzb_district_name", 1)
keys.put("shape", 1)
return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList()
}
static def queryAllRegionNameByCity(){
@@ -5522,4 +5538,10 @@ class CommonDatautils {
return 0
}
static def findBusinessByCity(city){
def q = new BasicDBObject()
q.put("short_city", city)
return bk_business_coll.find(q).toList()
}
}
@@ -0,0 +1,268 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.gj.xiamen.ElasticUtils89
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.looplineinfo.InsidePolygon
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.bson.types.ObjectId
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.support.WriteRequest
import org.elasticsearch.action.update.UpdateRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.xcontent.XContentType
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class BusinessQuarterProcess {
def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity()
def fullCityMap = CommonDatautils.queryFullNameCity()
DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020)
public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com")
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
static main(args) {
def task = new BusinessQuarterProcess()
task.run()
}
def run(){
// synchBusinessData()
// updateBusinessQuarter()
// addRegion()
synchBusinessToEs()
}
def addRegion(){
def list = CommonDatautils.queryRangeByCat("区域")
println list.size()
def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2")))
for(def d : dbcursor){
if(d.district){
continue
}
for(def regionShape : list){
if(d.short_city!=regionShape.city_short){
continue
}
def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates))
if(flag){
def udata = new BasicDBObject()
udata.put("district", regionShape.mzb_district_name)
udata.put("short_district", regionShape.district_short)
bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
break
}
}
}
}
def synchBusinessData(){
def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList()
synchMongoProcess(datas)
}
def synchMongoProcess(datas){
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(datas.size())
for(def i=0;i<datas.size();i++){
final def d = datas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
createData(d)
println d.city +" "+totalCount.get() +"/"+datas.size()
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
}
def createData(d){
def row = new BasicDBObject()
row.put("_id", d._id)
row.put("short_province", d.province)
def short_city = RegionData.formatCity(d.city)
row.put("short_city", short_city)
row.put("name", d.name)
row.put("bd_location", ["lon":DataHandleUtils.getDoubleRound(6,d.longitude), "lat":DataHandleUtils.getDoubleRound(6,d.latitude)])
def data = fullCityMap.get(short_city)
if(data){
row.put("province", data.province)
row.put("city", data.city)
}else{
println d.city + " 没找到简称"
}
def gdPoints = FangjiaClientUtils.convert(d.longitude+","+d.latitude, 3)
if(gdPoints){
row.put("location", ["lon":DataHandleUtils.getDoubleRound(6,gdPoints[0]), "lat":DataHandleUtils.getDoubleRound(6,gdPoints[1])])
}
def border = d.border
def bd_shapes = []
def gd_shapes = []
if(border){
def arrs = border.split(";")
if(border.contains("|")){
println d._id.toString()+" 可能含多个围栏"
}
for(def i=0;i<arrs.size();i++){
def str = arrs[i]
def arr = str.split(",")
def bd_lng = DataHandleUtils.getDoubleRound(6, arr[0].toString().toDouble())
def bd_lat = DataHandleUtils.getDoubleRound(6, arr[1].toString().toDouble())
if(!bd_shapes.contains([bd_lng, bd_lat]) ){
bd_shapes << [bd_lng, bd_lat]
}
if(bd_shapes.contains([bd_lng, bd_lat]) && i+1==arrs.size() && bd_shapes[0]==[bd_lng, bd_lat]){
bd_shapes << [bd_lng, bd_lat]
}
def tempGdPoints = FangjiaClientUtils.convert(bd_lng+","+bd_lat, 3)
if(tempGdPoints){
def gdlist = [DataHandleUtils.getDoubleRound(6,tempGdPoints[0]), DataHandleUtils.getDoubleRound(6,tempGdPoints[1])]
if(!gd_shapes.contains(gdlist)) {
gd_shapes << gdlist
}
if(gd_shapes.contains(gdlist) && i+1==arrs.size() && gd_shapes[0]==gdlist){
gd_shapes << gdlist
}
}
}
if(bd_shapes[0] != bd_shapes[bd_shapes.size()-1]){
bd_shapes << bd_shapes[0]
gd_shapes << gd_shapes[0]
}
}
def gdMap = [:]
gdMap.put("type" , "polygon")
gdMap.put("coordinates", [gd_shapes])
row.put("shape", gdMap)
def bdMap = [:]
bdMap.put("type" , "polygon")
bdMap.put("coordinates", [bd_shapes])
row.put("bd_shape", bdMap)
row.put("c_date", new Date())
bk_business_coll.update(new BasicDBObject("_id", row._id), new BasicDBObject('$set', row),true, false)
}
def updateBusinessQuarter(){
def citys = CommonDatautils.findAllCity()
for(def c : citys){
if(["上海","北京"].contains(c)){
continue
}
def cityBusinessList = CommonDatautils.findBusinessByCity(c)
def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region","name", "business_quarter", "lat", "lng"])
businessProcess(c, districts, cityBusinessList)
}
}
def businessProcess(c,districts, cityBusinessList){
def updateIndex = 0
for(def d : districts){
if(!d.business_quarter){
def businessQuarter = findBusinessByRange(d, cityBusinessList)
if(businessQuarter){
updateIndex ++
def udata = new BasicDBObject()
udata.put("business_quarter", businessQuarter)
udata.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata))
}
}
}
println c +" "+ districts.size() +" "+updateIndex
}
def findBusinessByRange(d, cityBusinessList){
def name= null
for(def shapeObj : cityBusinessList){
if(d.lng){
def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(shapeObj.shape.coordinates))
if(flag){
name = shapeObj.name
break
}
}
}
return name
}
def synchBusinessToEs(){
def dbcursor = bk_business_coll.find(new BasicDBObject())
def rows = []
def index = 0
for(def d : dbcursor){
index++
if(index<5000){
continue
}
// rows << d
// if(rows.size()>=100){
// updateByEs(rows)
// rows.clear()
// }
updateByEs(d, index)
}
}
def updateByEs(row, index){
BulkRequest request = new BulkRequest();
def client = ElasticUtils89.getClient("192.168.0.89")
int processedCount = 0;
def id = row.get("_id").toString()
def doc = [:]
doc.put("province",row.province)
doc.put("short_province",row.short_province)
doc.put("city",row.city)
doc.put("short_city",row.short_city)
doc.put("district",row.district)
doc.put("short_district",row.short_district)
doc.put("name", row.name)
doc.put("location", row.location)
doc.put("bd_location", row.bd_location)
doc.put("shape", row.shape)
doc.put("bd_shape", row.bd_shape)
request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON));
if (request.numberOfActions() >= 0) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT);
if (bulkResponse.hasFailures()) {
// 处理错误
println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage()
} else {
processedCount += request.numberOfActions();
// println "Processed " + processedCount + " documents"
}
}
}
}
@@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.rabbitmq.client.impl.AMQImpl
import org.bson.types.ObjectId
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.text.SimpleDateFormat
import java.util.concurrent.CopyOnWriteArrayList
@@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{
//"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房"
//
def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房",
def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房",
"乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源
@@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{
"car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构",
"build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"]
final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018)
final static def EMAILS = [
"gaojuan@fangjia.com"
@@ -115,18 +118,12 @@ class DistrictFieldAddBatch extends BaseBatch{
headers << fieldsNameMap.get(it)
}
headers << "抓取量"
headers << "匹配量"
headers << "0.9分以上量"
headers << "匹配率"
headers << "0.9以上匹配率"
// analysRets << headers
for(def c : citys){
def totalCount = CommonDatautils.findAllDistrictCount("district", c)
if(totalCount){
// rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合
analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配
analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表
}
}
println analysRets.toString()
@@ -139,8 +136,10 @@ class DistrictFieldAddBatch extends BaseBatch{
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findEsDatas(c, esWebSource)
def matchRets1 = matchApi(esRets)
def matchRets2 = selectRestByCredit(matchRets1)
println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size()
def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource)
@@ -172,6 +171,65 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def addFieldFromMatchMapping(c, totalCount, analysRets){
def fields = fieldsMap.keySet() as List
sourceMap.each { seaweedmapSource, esWebSource ->
def esRets = findMatchSameDatas(c, esWebSource)
println esWebSource +" 新老算法匹配相同量:"+esRets.size()
def updateRets = processFieldsFromMatch(c , esRets, esWebSource)
def precentMap = fieldUpdate(totalCount, updateRets)
println c +" "+precentMap.toString()
updateDB(c, updateRets)
println "------------------------------------------------------------------------------"
def analyRet = []
analyRet << seaweedmapSource
analyRet << c
fields.each{key->
analyRet << precentMap.get(key)
}
analyRet << esRets.size()
analysRets << analyRet
}
return analysRets;
}
def findMatchSameDatas(c, esWebSource){
def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true))
def siteDatas = thirdSeaweedMappingColl.find(query).toList()
def rows = new CopyOnWriteArrayList()
def totalCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(siteDatas.size())
for(def i=0;i<siteDatas.size();i++){
final def d = siteDatas[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
totalCount.addAndGet(1)
def obj = findCommunities(d.url)
if(obj){
obj.match_id = d.match_id
rows << obj
}
// println d.city +" "+totalCount.get() +"/"+siteDatas.size()
} catch (Exception e) {
e.printStackTrace()
println d.city +" "+d.name
} finally {
latch.countDown()
}
}
});
}
latch.await()
return rows
}
def addFieldFromRonghe(c, totalCount, rh_analysRets){
@@ -241,14 +299,10 @@ class DistrictFieldAddBatch extends BaseBatch{
fields << "city"
fields << "estate_type2"
def fieldValueMap = [:]
def rets = []
rets << ["第三方城市","第三方区域","第三方名称","第三方地址","第三方户数","匹配区域","匹配名称","匹配可信度","方式"]
for(def matchDist : matchRets){
// if(!["智富中心","龙山新寓","天怡瀚园","远盛凤林府","京口路小区","梦溪路小区","浩创优尚","溪山丽苑","旭园"].contains(matchDist.name)){
// continue
// }
def dist = CommonDatautils.findDistrictFieldById(matchDist.match_id, fields)
if(!dist){
println c+" "+matchDist.match_id+" 不存在"
@@ -256,7 +310,6 @@ class DistrictFieldAddBatch extends BaseBatch{
}
def newValues = strandField(c, dist, matchDist)
if(newValues){
rets << [matchDist.city, matchDist.region, matchDist.name, matchDist.address,newValues.get("household_count"), matchDist.match_region, matchDist.match_name, matchDist.credit, matchDist.flag]
println dist._id.toString()+" "+newValues.toString()
fieldValueMap.put(dist._id.toString(), newValues)
}
@@ -662,4 +715,26 @@ class DistrictFieldAddBatch extends BaseBatch{
jms.sendMail(email, "字段填充情况分析", sb.toString()+rh_sb.toString());
}
}
def findCommunities(url){
if(!url){
return null
}
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("url", url))
def rets = []
def datas = null
try{
datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "community", queryBuilder);
}catch(Exception e){
findCommunities(url)
}
def obj = null
if(datas) {
for (int k = 0; k < datas.size(); k++) {
def d = datas[k];
obj = d
}
}
return obj;
}
}
@@ -52,8 +52,8 @@ class AddDistImagesBatch extends BaseBatch{
}
def findAllEsData2(city, mdate){
QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(mdate.getTime()).includeLower(true)
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "安居客小程序"))
//安居客小程序
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("source", "贝壳找房"))
.must(date_range).must(date_range)
def rets = []
@@ -110,7 +110,7 @@ class AddDistImagesBatch extends BaseBatch{
try{
addCount.addAndGet(1)
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","安居客").append("match_id", new BasicDBObject('$exists', true))
def query = new BasicDBObject("city", d.city).append("district",d.region).append("name",d.name).append("source","贝壳找房").append("match_id", new BasicDBObject('$exists', true))
def obj = thirdSeaweedMappingColl.findOne(query)
if(obj) {
def imageObj = seaweed_fs_coll.findOne(new ObjectId(obj.match_id))