From 0cd2aa435361a595c15705b48bd4b17ce5a295a1 Mon Sep 17 00:00:00 2001 From: majintao Date: Tue, 11 Feb 2025 10:45:23 +0800 Subject: [PATCH 1/5] =?UTF-8?q?=E5=8C=97=E4=BA=A4=E6=89=80=E6=8E=A5?= =?UTF-8?q?=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ComprehensiveQueryController.java | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java index 1088d4648..40afe3fe6 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/ComprehensiveQueryController.java @@ -228,7 +228,7 @@ public class ComprehensiveQueryController { String cityStr = model.getCity(); if (StringUtils.isNotBlank(cityStr) && cityStr.matches("\\d+")) { AdministrativeDivision administrativeDivision = administrativeDivisionService.findcityByCode(cityStr); - if(administrativeDivision == null){ + if (administrativeDivision == null) { mav.setModel(BasicModel.buildBadRequestModel("城市代码错误")); return mav; } @@ -289,11 +289,11 @@ public class ComprehensiveQueryController { // return mav; // } JSONObject result = matchResponse.getJSONObject("result"); - if(!result.containsKey("id")){ + if (!result.containsKey("id")) { mav.setModel(BasicModel.buildBadRequestModel("该地址不支持估价!")); return mav; } - String id = matchResponse.getJSONObject("result") .getString("id"); + String id = matchResponse.getJSONObject("result").getString("id"); District district = districtService.findById(id); if (district == null) { mav.setModel(BasicModel.buildBadRequestModel("该地址不支持估价!")); @@ -419,9 +419,11 @@ public class ComprehensiveQueryController { if (facilitys != null) { List hospitalList = facilitys.getHospital(); List hospitals = new ArrayList(); - for (String hospital : hospitalList) { - if (hospital.endsWith("医院")) { - hospitals.add(hospital); + if (hospitalList != null && !hospitalList.isEmpty()) { + for (String hospital : hospitalList) { + if (hospital.endsWith("医院")) { + hospitals.add(hospital); + } } } facilitys.setHospital(hospitals); @@ -753,7 +755,7 @@ public class ComprehensiveQueryController { districtPriceMap.put("name", district.getName()); // 估值中已经取过评估2B价格,且经过系数调整,不能在覆盖 districtPriceMap.put("avgPrice", evaluation.getAvgPrice()); - Long totalPrice = new BigDecimal((Integer)districtPriceMap.get("avgPrice")).multiply(new BigDecimal(evaluation.getArea())).longValue(); + Long totalPrice = new BigDecimal((Integer) districtPriceMap.get("avgPrice")).multiply(new BigDecimal(evaluation.getArea())).longValue(); districtPriceMap.put("totalPrice", totalPrice); if (model.getScope() == EVALUATE_SHOW_DETAIL) { From cf3fbdcc803a89373b19b67d301ce5fca3756be7 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 11 Feb 2025 16:12:02 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=AD=97=E6=AE=B5?= =?UTF-8?q?=E8=87=AA=E5=8A=A8=E5=A1=AB=E5=85=85=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 22 ++ .../data_handle/BusinessQuarterProcess.groovy | 268 ++++++++++++++++++ .../gj/district/DistrictFieldAddBatch.groovy | 103 ++++++- .../gj/images/AddDistImagesBatch.groovy | 6 +- 4 files changed, 382 insertions(+), 17 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BusinessQuarterProcess.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 94a3445f5..82cffedbd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -103,6 +103,8 @@ class CommonDatautils { public static final DBCollection STATS_INDEX_COLL = MongoUtils.getCollectionByName("fangjia_stat", "stats_index", "mongo.db.fangjia.com") + public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com") + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -4967,6 +4969,20 @@ class CommonDatautils { return regionMap } + static def queryRangeByCat(cat){ + def q = new BasicDBObject() + q.put("type",cat) + q.put("shape",new BasicDBObject('$exists', true)) + def keys = new BasicDBObject() + keys.put("city_short",1) + keys.put("district_short",1) + keys.put("city",1) + keys.put("district",1) + keys.put("mzb_district_name", 1) + keys.put("shape", 1) + return ADMINISTRATIVE_MAPPING_COLL.find(q,keys).toList() + } + static def queryAllRegionNameByCity(){ @@ -5522,4 +5538,10 @@ class CommonDatautils { return 0 } + static def findBusinessByCity(city){ + def q = new BasicDBObject() + q.put("short_city", city) + return bk_business_coll.find(q).toList() + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BusinessQuarterProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BusinessQuarterProcess.groovy new file mode 100644 index 000000000..f122be3c0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/BusinessQuarterProcess.groovy @@ -0,0 +1,268 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.action.update.UpdateRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType + +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class BusinessQuarterProcess { + + def regionMap = com.fangjia.script.commData.CommonDatautils.queryAllRegionNameByCity() + def fullCityMap = CommonDatautils.queryFullNameCity() + DBCollection bk_map_bussiness_COLL = MongoUtils.getCollectionByName("crawl", "bk_map_sq", "192.168.0.206",27020) + + public static final DBCollection bk_business_coll = MongoUtils.getCollectionByName("fangjia_base", "bk_business", "mongo.db.fangjia.com") + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + + static main(args) { + def task = new BusinessQuarterProcess() + task.run() + } + def run(){ +// synchBusinessData() +// updateBusinessQuarter() +// addRegion() + synchBusinessToEs() + } + + def addRegion(){ + def list = CommonDatautils.queryRangeByCat("区域") + println list.size() + def dbcursor = bk_business_coll.find(new BasicDBObject("_id", new ObjectId("67a318893b75f388c80979f2"))) + for(def d : dbcursor){ + if(d.district){ + continue + } + for(def regionShape : list){ + if(d.short_city!=regionShape.city_short){ + continue + } + def flag = InsidePolygon.isInArea(d.location.lon, d.location.lat, InsidePolygon.createPoint4(regionShape.shape.coordinates)) + if(flag){ + def udata = new BasicDBObject() + udata.put("district", regionShape.mzb_district_name) + udata.put("short_district", regionShape.district_short) + bk_business_coll.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + break + } + } + } + } + + def synchBusinessData(){ + def datas = bk_map_bussiness_COLL.find(new BasicDBObject("type", null)).toList() + synchMongoProcess(datas) + + } + + def synchMongoProcess(datas){ + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(datas.size()) + for(def i=0;i=100){ +// updateByEs(rows) +// rows.clear() +// } + updateByEs(d, index) + } + } + + def updateByEs(row, index){ + BulkRequest request = new BulkRequest(); + def client = ElasticUtils89.getClient("192.168.0.89") + int processedCount = 0; + + def id = row.get("_id").toString() + def doc = [:] + doc.put("province",row.province) + doc.put("short_province",row.short_province) + doc.put("city",row.city) + doc.put("short_city",row.short_city) + doc.put("district",row.district) + doc.put("short_district",row.short_district) + doc.put("name", row.name) + doc.put("location", row.location) + doc.put("bd_location", row.bd_location) + doc.put("shape", row.shape) + doc.put("bd_shape", row.bd_shape) + + + request.add(new IndexRequest("bk_business", "_doc", id).source(doc, XContentType.JSON)); + if (request.numberOfActions() >= 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "[${index}] Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); +// println "Processed " + processedCount + " documents" + } + } + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index e2625366c..83f747d21 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -18,6 +18,8 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders import java.text.SimpleDateFormat import java.util.concurrent.CopyOnWriteArrayList @@ -39,7 +41,7 @@ class DistrictFieldAddBatch extends BaseBatch{ //"安居客":"安居客小程序","链家":"链家","幸福里":"幸福里","城市房产":"城市房产小程序","搜房":"搜房" // - def sourceMap = ["幸福里":"幸福里APP","城市房产":"城市房产小程序","安居客小程序":"安居客小程序","安居客移动端":"安居客移动端","贝壳找房":"贝壳找房","安居客-新房":"安居客-新房","安居客":"安居客","链家":"链家","贝壳找房-新房":"贝壳找房-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情","链家":"链家","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房APP","贝壳找房":"贝壳找房", + def sourceMap = ["贝壳找房":"贝壳找房","链家":"链家","贝壳找房-新房":"贝壳找房-新房","安居客":"安居客","幸福里":"幸福里","安居客小程序":"安居客小程序","城市房产":"城市房产小程序","安居客移动端":"安居客移动端","安居客-新房":"安居客-新房","楼盘网-新房":"楼盘网-新房","禧泰":"中国房价行情", "城市房产":"城市房产","搜房":"搜房","我爱我家":"我爱我家","诸葛找房":"诸葛找房", "乐有家":"乐有家","吉屋":"吉屋","C21":"C21","365淘房":"365淘房"] //key:穆融合表的来源 valve:es解析表的来源 @@ -61,6 +63,7 @@ class DistrictFieldAddBatch extends BaseBatch{ "car_site_count":"车位","car_site":"车位描述","property_life":"产权年限","address":"地址","build_structure":"建筑结构", "build_type":"建筑类型", "land_property":"权属类别","live_date":"入住时间","loop_line_info":"环线"] + final DBCollection thirdSeaweedMappingColl = MongoUtils.getCollectionByName("fangjia_craw", "third_seaweed_mapping", "192.168.0.235", 27018) final static def EMAILS = [ "gaojuan@fangjia.com" @@ -115,18 +118,12 @@ class DistrictFieldAddBatch extends BaseBatch{ headers << fieldsNameMap.get(it) } - headers << "抓取量" - headers << "匹配量" - headers << "0.9分以上量" - headers << "匹配率" - headers << "0.9以上匹配率" -// analysRets << headers - for(def c : citys){ def totalCount = CommonDatautils.findAllDistrictCount("district", c) if(totalCount){ // rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合 - analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配 +// analysRets = addFieldFromMatch(c, totalCount, analysRets)//匹配 + analysRets = addFieldFromMatchMapping(c, totalCount, analysRets)//新旧匹配中间表 } } println analysRets.toString() @@ -139,8 +136,10 @@ class DistrictFieldAddBatch extends BaseBatch{ sourceMap.each { seaweedmapSource, esWebSource -> def esRets = findEsDatas(c, esWebSource) + def matchRets1 = matchApi(esRets) def matchRets2 = selectRestByCredit(matchRets1) + println esWebSource +" 抓取量:"+esRets.size()+" 匹配量:"+matchRets1.size()+" 0.9分以上量:"+matchRets2.size() def updateRets = processFieldsFromMatch(c , matchRets2, esWebSource) @@ -172,6 +171,65 @@ class DistrictFieldAddBatch extends BaseBatch{ } + def addFieldFromMatchMapping(c, totalCount, analysRets){ + def fields = fieldsMap.keySet() as List + + sourceMap.each { seaweedmapSource, esWebSource -> + def esRets = findMatchSameDatas(c, esWebSource) + println esWebSource +" 新老算法匹配相同量:"+esRets.size() + + def updateRets = processFieldsFromMatch(c , esRets, esWebSource) + + def precentMap = fieldUpdate(totalCount, updateRets) + println c +" "+precentMap.toString() + updateDB(c, updateRets) + println "------------------------------------------------------------------------------" + + def analyRet = [] + analyRet << seaweedmapSource + analyRet << c + fields.each{key-> + analyRet << precentMap.get(key) + } + analyRet << esRets.size() + analysRets << analyRet + } + return analysRets; + + } + + + def findMatchSameDatas(c, esWebSource){ + def query = new BasicDBObject("city", c).append("source", esWebSource).append("match_id", new BasicDBObject('$exists', true)).append("old_match_id", new BasicDBObject('$exists',true)) + def siteDatas = thirdSeaweedMappingColl.find(query).toList() + def rows = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(siteDatas.size()) + for(def i=0;i Date: Wed, 12 Feb 2025 17:20:16 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E5=88=AB=E5=90=8D?= =?UTF-8?q?=E8=B7=91=E4=B9=9D=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddrAlaisProcess.groovy | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy new file mode 100644 index 000000000..1cf20acc9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.ExcelUtils + +class AddrAlaisProcess { + + static main(args) { + def task = new AddrAlaisProcess() +// task.run() + task.nineApi() + } + + def nineApi(){ + def fileName = "/home/goojia/gj/district/addr_alias.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-测试.xlsx" + } + + ExcelUtils.multiThreadingModify(fileName, {d-> + def addrMap = nineAddrSplit(d['B'], d['E']) + def addrList = [] + if(addrMap && addrMap.address){ + d['G'] = addrMap.city + d['H'] = addrMap.address + addrList << addrMap.address + } + if(d['F']){ + def addrs = d['F'].split(",") + def addr_alias = [] + for(def addr : addrs){ + def map = nineAddrSplit(d['B'], addr) + if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){ + if(!addrList.contains(map.address)){ + addrList << map.address + addr_alias << map.address + } + } + } + d['I'] = addr_alias.join(",") + } + }) + + } + + def nineAddrSplit(city, addr){ + def jsonObject = com.fangjia.script.gj.district.FangjiaClientUtils.formatNinelevel(city, addr) + def address = "" + def map = [:] + if (jsonObject) { + map.province = jsonObject.get("province") + map.city = jsonObject.get("city") + map.district = jsonObject.get("district") + if (jsonObject.get("street") && jsonObject.get("street") != "null") { + address = jsonObject.get("street") + } + if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") { + address = address + jsonObject.get("streetNumber") + } + map.address = address + } + return map + } + + def run(){ + def citys = CommonDatautils.findAllCity() + def rows = [] + citys.each{c-> + def districts = CommonDatautils.findAllDistricts(c, "district", ["city","region", "name","address", "addr_alias"]) + println c +" "+districts.size() + def num = 0 + for(def d : districts){ + num++ + def row = [num, d.city, d.region, d.name, d.address] + if(d.addr_alias){ + row << d.addr_alias.join(",") + } + rows << row + } + } +// ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区") + FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.bat", null, "utf-8", false, "#",""); + } +} From 683de63a610ed659d6a21da6eacf6a1465317e1b Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 12 Feb 2025 18:09:51 +0800 Subject: [PATCH 4/5] update --- .../gj/district/DistrictFieldAddBatch.groovy | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy index 83f747d21..5610ae238 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/DistrictFieldAddBatch.groovy @@ -119,6 +119,7 @@ class DistrictFieldAddBatch extends BaseBatch{ } for(def c : citys){ + def totalCount = CommonDatautils.findAllDistrictCount("district", c) if(totalCount){ // rh_analysRets = addFieldFromRonghe(c, totalCount, rh_analysRets)//融合 @@ -397,12 +398,16 @@ class DistrictFieldAddBatch extends BaseBatch{ newValues.put(seaweedfield, newVal) } if(seaweedfield == "lng"){ - def locationMap = FiledStrand.processLngAndLat(esVal) - if(locationMap){ - newValues.put("lat", locationMap.lat) - newValues.put("lng", locationMap.lng) - newValues.put("lat2", locationMap.lat2) - newValues.put("lng2", locationMap.lng2) + try{ + def locationMap = FiledStrand.processLngAndLat(esVal) + if(locationMap){ + newValues.put("lat", locationMap.lat) + newValues.put("lng", locationMap.lng) + newValues.put("lat2", locationMap.lat2) + newValues.put("lng2", locationMap.lng2) + } + }catch(Exception e){ + println esVal } } } From 8447b62637072b6cfc12f264b333747583cfe484 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Thu, 13 Feb 2025 09:47:30 +0800 Subject: [PATCH 5/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddrAlaisProcess.groovy | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy index 1cf20acc9..62d1f299e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy @@ -50,9 +50,21 @@ class AddrAlaisProcess { def address = "" def map = [:] if (jsonObject) { - map.province = jsonObject.get("province") - map.city = jsonObject.get("city") - map.district = jsonObject.get("district") + if (jsonObject.get("province") && jsonObject.get("province") != "null") { + map.province = jsonObject.get("province") + }else{ + map.province = "" + } + if (jsonObject.get("city") && jsonObject.get("city") != "null") { + map.city = jsonObject.get("city") + }else{ + map.city = "" + } + if (jsonObject.get("district") && jsonObject.get("district") != "null") { + map.district = jsonObject.get("district") + }else{ + map.district = "" + } if (jsonObject.get("street") && jsonObject.get("street") != "null") { address = jsonObject.get("street") }