diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94a3445f5..82cffedbd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -103,6 +103,8 @@ class CommonDatautils { public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") + public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -4967,6 +4969,20 @@ class CommonDatautils { return regionMap } + static def queryRangeByCat(cat){ + def q = new BasicDBObject() + q.put("type",cat) + q.put("shape",new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() + keys.put("city_short",1) + keys.put("district_short",1) + keys.put("city",1) + keys.put("district",1) + keys.put("mzb_district_name", 1) + keys.put("shape", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList() + } + static def queryAllRegionNameByCity(){ @@ -5522,4 +5538,10 @@ class CommonDatautils { return 0 } + static def findBusinessByCity(city){ + def q = new BasicDBObject() + q.put("short_city", city) + return bk_business_coll.find(q).toList() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BusinessQuarterProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BusinessQuarterProcess.groovy new file mode 100644 index 000000000..f122be3c0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BusinessQuarterProcess.groovy @@ -0,0 +1,268 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class BusinessQuarterProcess { + + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def fullCityMap = CommonDatautils.queryFullNameCity() + DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020) + + public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + static main(args) { + def task = new BusinessQuarterProcess() + task.run() + } + def run(){ +// synchBusinessData() +// updateBusinessQuarter() +// addRegion() + synchBusinessToEs() + } + + def addRegion(){ + def list = CommonDatautils.queryRangeByCat("区域") + println list.size() + def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2"))) + for(def d : dbcursor){ + if(d.district){ + continue + } + for(def regionShape : list){ + if(d.short_city!=regionShape.city_short){ + continue + } + def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates)) + if(flag){ + def udata = new BasicDBObject() + udata.put("district", regionShape.mzb_district_name) + udata.put("short_district", regionShape.district_short) + bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + break + } + } + } + } + + def synchBusinessData(){ + def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList() + synchMongoProcess(datas) + + } + + def synchMongoProcess(datas){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i=100){ +// updateByEs(rows) +// rows.clear() +// } + updateByEs(d, index) + } + } + + def updateByEs(row, index){ + BulkRequest request = new BulkRequest(); + def client = ElasticUtils89.getClient("192.168.0.89") + int processedCount = 0; + + def id = row.get("_id").toString() + def doc = [:] + doc.put("province",row.province) + doc.put("short_province",row.short_province) + doc.put("city",row.city) + doc.put("short_city",row.short_city) + doc.put("district",row.district) + doc.put("short_district",row.short_district) + doc.put("name", row.name) + doc.put("location", row.location) + doc.put("bd_location", row.bd_location) + doc.put("shape", row.shape) + doc.put("bd_shape", row.bd_shape) + + + request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON)); + if (request.numberOfActions() >= 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); +// println "Processed " + processedCount + " documents" + } + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index e2625366c..83f747d21 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders import java.text.SimpleDateFormat import java.util.concurrent.CopyOnWriteArrayList @@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{ //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" // - def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", + def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房", "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源 @@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{ "car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构", "build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"] + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) final static def EMAILS = [ "gaojuan@fangjia.com" @@ -115,18 +118,12 @@ class DistrictFieldAddBatch extends BaseBatch{ headers << fieldsNameMap.get(it) } - headers << "抓取量" - headers << "匹配量" - headers << "0.9分以上量" - headers << "匹配率" - headers << "0.9以上匹配率" -// analysRets << headers - for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) if(totalCount){ // rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合 - analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配 +// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配 + analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表 } } println analysRets.toString() @@ -139,8 +136,10 @@ class DistrictFieldAddBatch extends BaseBatch{ sourceMap.each { seaweedmapSource, esWebSource -> def esRets = findEsDatas(c, esWebSource) + def matchRets1 = matchApi(esRets) def matchRets2 = selectRestByCredit(matchRets1) + println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size() def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource) @@ -172,6 +171,65 @@ class DistrictFieldAddBatch extends BaseBatch{ } + def addFieldFromMatchMapping(c, totalCount, analysRets){ + def fields = fieldsMap.keySet() as List + + sourceMap.each { seaweedmapSource, esWebSource -> + def esRets = findMatchSameDatas(c, esWebSource) + println esWebSource +" 新老算法匹配相同量:"+esRets.size() + + def updateRets = processFieldsFromMatch(c , esRets, esWebSource) + + def precentMap = fieldUpdate(totalCount, updateRets) + println c +" "+precentMap.toString() + updateDB(c, updateRets) + println "------------------------------------------------------------------------------" + + def analyRet = [] + analyRet << seaweedmapSource + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analysRets << analyRet + } + return analysRets; + + } + + + def findMatchSameDatas(c, esWebSource){ + def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true)) + def siteDatas = thirdSeaweedMappingColl.find(query).toList() + def rows = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(siteDatas.size()) + for(def i=0;i