From a094b9ead81c231d5cfa5fc4c7170ce4070ce1df Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 2 Apr 2025 09:45:32 +0800 Subject: [PATCH 01/13] update --- .../script/ccbevaluation/AddSzDealData.groovy | 2 +- .../fangjia/script/ccbevaluation/Test.groovy | 11 +- .../script/commData/CommonDatautils.groovy | 14 +- .../script/gj/FangjiatongUserStat.groovy | 61 +++- .../com/fangjia/script/gj/FiledStrand.groovy | 179 ++++++++-- .../gj/data_handle/AddDevShapeData.groovy | 206 ++++++++++- .../gj/data_handle/AddrAlaisProcess.groovy | 114 ++++-- .../data_handle/AutoAddDistrictBatch.groovy | 42 ++- .../gj/data_handle/AutoAddOfficeBatch.groovy | 84 ++++- .../gj/data_handle/DevUpdateTestBatch.groovy | 4 +- .../script/gj/data_handle/MinglujiData.groovy | 8 +- .../gj/data_handle/RegionStandardBatch.groovy | 4 +- .../gj/data_handle/TownMiddleProcess.groovy | 12 +- .../gj/data_handle/TownStandardBatch.groovy | 329 +++++++++++++----- .../data_handle/TownshipStrandUpdate.groovy | 74 +++- .../gj/district/FangjiaClientUtils.groovy | 4 +- .../gj/district/OfficeFieldCleanBatch.groovy | 205 +++++++++-- .../script/gj/es/EsMonthHouseStat.groovy | 2 +- .../fangjia/script/gj/poc/AddPriceTest.groovy | 46 ++- .../script/gj/village/govLocation.groovy | 2 +- .../script/gj/xiamen/SeaweedDataSynch.groovy | 92 ++++- 21 files changed, 1264 insertions(+), 231 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy index dfd784e71..4b9781f02 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/AddSzDealData.groovy @@ -8,7 +8,7 @@ import com.mongodb.DBCollection /** * 深圳公众号成交数据录入 */ -def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2024年11月深圳二手房成交.xlsx", 0, 1) +def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\成交\\2025年01月深圳二手房成交的.xlsx", 0, 1) def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 3ccdc43ef..ce47aa8db 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -1353,13 +1353,14 @@ def findAvgPrice(def commid, def pgsqlInstance, def cityid){ //},0) -def fileName = "D:\\数据\\建行\\房产估值\\指导价\\20241025人工指导价.xlsx" +def fileName = "D:\\数据\\建行\\房产估值\\指导价\\更新人工指导价_0314.xlsx" ExcelUtils.multiThreadingModify(fileName, { d -> - def dist = CommonDatautils.findDistrictById(d['J']) + def dist = CommonDatautils.findDistrictById(d['A']) if(dist){ - d['A'] = dist.city - d['B'] = dist.region - d['C'] = dist.name + d['C'] = dist.city + d['D'] = dist.region + d['E'] = dist.name + d['F'] = dist.status } }) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index e6e2d944b..952f3eccd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -262,7 +262,7 @@ class CommonDatautils { q.put("status", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","托斯卡纳庄园") +// q.put("name","新泾三村") DBObject keys = new BasicDBObject(); if(fields){ @@ -821,7 +821,9 @@ class CommonDatautils { def dbsour = SEAWEED_COLL.find(q, keys).batchSize(5000) def rets = [] for(def d : dbsour){ - rets << d + if(!d.township){ + rets << d + } } return rets; } @@ -4717,6 +4719,14 @@ class CommonDatautils { } + + static def httpSaveData89(def newUrl,def params){ + def header = [:] + header.put("Authorization","Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString()) + doPost(newUrl,header, params, true, "utf-8") + } + + static def doPost(def targetUrl, def headers, def params, def isJson, def charset){ StringBuilder buffer = new StringBuilder(); try{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FangjiatongUserStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FangjiatongUserStat.groovy index 3932cc7d9..29f729c63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FangjiatongUserStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FangjiatongUserStat.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj +import com.fangjia.script.common.DataHandleUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection; @@ -12,12 +13,13 @@ import com.fangjia.script.util.ExcelUtils; class FangjiatongUserStat { public static final DBCollection FJT_USER_COLL = MongoUtils.getCollectionByName("fangjiatong_log", "utm_gif_log", "222.73.242.42", 52002) - static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://222.73.242.42:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); + static final def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/goojia_fangjia?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); public static void main(String[] args) { def task = new FangjiatongUserStat(); - task.run(); +// task.run(); + task.stat() } def run(){ @@ -60,4 +62,59 @@ class FangjiatongUserStat { return ret; } + def stat(){ + def users = findUses(12) + def months = ["2019-01-01","2019-02-01","2019-03-01","2019-04-01","2019-05-01","2019-06-01","2019-07-01","2019-08-01","2019-09-01","2019-10-01","2019-11-01","2019-12-01","2020-01-01"] + def rets = [] + def header = ["用户id","支行名称", "备注名称"] + months.each{m-> + header << m.substring(0, 7) + } + rets << header + for(def m : users){ + def ret = [] + ret << m.user_id + ret << m.name + ret << m.org_name + for(def i=1;i=5000 && v<=10000000){ + if(v>=1000 && v<=10000000){ return v.toString().toDouble().toInteger() } } @@ -288,11 +285,25 @@ class FiledStrand { return null } + + + static def processLobbyHeight(height){ + if(height){ + def v = getValue(height) + if(v>=2.4 && v<=30){ + return v.toString() + } + } + return null + } + static def processElevators(elevators){ if(elevators){ def v = getValue(elevators) - if(v>0 && v<=100){ - return v.toString() + if(v instanceof Integer){ + if(v>0 && v<=100){ + return v.toString() + } } } return null @@ -317,24 +328,34 @@ class FiledStrand { static def processStandardArea(area){ if(area){ - area = area.toString().replace(",",",").replace("㎡", "").replace("m", "").replace("/","") - area = Pattern.compile("[\\u4e00-\\u9fa5]").matcher(area).replaceAll(""); - return area + def v = getValue(area) + if(v>=500 && v<=20000){ + return v.toString().toDouble().toInteger() + } } return null } static def processCarSiteRent(rent){ if(rent){ - def v = getNumberRange(rent) - if(v){ - if(v=="0"){ + if(["-","0元/月","无","不详","未知","待查","免费元/月","/","0","*","待确认","undefined元/月","0元","0元每月","未知元/月","0.00元/月","未定","暂定","测算中","无元/月","未定元/月","另议元/月","面议元/月","面议元/月","有元/月","暂时未确定","0元(无停车位)"," 元/月"].contains(rent)){ + return null + }else if(rent.contains("暂无") || rent.contains("车位不归") || rent.contains("待定") || rent.contains("暂定") || rent.contains("暂完善") || rent.contains("待审批") || rent.contains("不详") || rent.contains("未定价格") || rent.contains("中国电信") || rent.contains("面谈")){ + return null + }else{ + rent = rent.replace("元/月","元/月").replace(",","").replace("#","").replace(".00","") + if(rent.toString().length()>100){ return null - }else{ - return v.toString() } + if(rent instanceof Double && rent.toDouble()<100){ + return null + } + + if(rent instanceof Integer && rent.toInteger()<100){ + return null + } + return rent } - return null } return null } @@ -382,7 +403,7 @@ class FiledStrand { static def processCarSite(car_site_count){ def v = getValue(car_site_count.toString()); if(v){ - if(v>5 && v<=100000){ + if(v>5 && v<=10000){ try { return v.toString().toInteger(); }catch(Exception e){ @@ -533,12 +554,82 @@ class FiledStrand { return null } + static def processLevel(val){ + def level = null + if (val) { + if(val.contains("超甲") || val.contains("5A") || val.contains("6A") || val.contains("顶级")){ + level = "超甲级" + }else if(val.indexOf("甲")>-1 || val.contains("A")){ + level = "甲级" + }else if(val.indexOf("乙")>-1 || val.contains("B")){ + level = "乙级"; + }else if(val.indexOf("丙")>-1 || val.contains("C")){ + level = "丙级"; + } + } + return level + } + + + static def processLoopLine(val){ + def line = null + if (val) { + if(!val.contains("暂") && !val.contains("无")){ + if(val.indexOf("环")>-1){ + line = val + } + } + } + return line + } + + static def processForeign(val){ + def foreign = null + if (val) { + if(val.indexOf("是")>-1){ + foreign = "是" + }else if(val.indexOf("否")>-1){ + foreign = "否"; + } + } + return foreign + } + + static def processRegister(val){ + def register = null + if (val) { + if(val.contains("不可以注册") || val.contains("不能注册")){ + register = "否"; + }else if(val.contains("可以注册")){ + register = "是" + }else if(val.indexOf("是")>-1){ + register = "是" + }else if(val.indexOf("否")>-1){ + register = "否"; + } + } + return register + } + + static def processFitment(val){ + def fitment = null + if(val){ + if(val.indexOf("精装")>-1){ + fitment = "精装" + }else if(val.indexOf("简单")>-1 || val.indexOf("简装")>-1 || val.indexOf("一般")>-1 || val.indexOf("普通")>-1){ + fitment = "简装"; + }else if(val.indexOf("毛坯")>-1){ + fitment = "毛坯"; + } + } + return fitment + } static def processLngAndLat(val) { - if (val) { + if (val && val!="null") { def map = [:] - def lng = DataHandleUtils.getDoubleRound(6, val.x) - def lat = DataHandleUtils.getDoubleRound(6, val.y) + def lng = DataHandleUtils.getDoubleRound(6, val.lon) + def lat = DataHandleUtils.getDoubleRound(6, val.lat) Long gd_lat = lat * 10000000000L Long gd_lng = lng * 10000000000L map.lat = gd_lat @@ -618,7 +709,8 @@ class FiledStrand { if(cubage_percent){ def v = getValue(cubage_percent) if(v){ - if(v>=0.3 && v<=10){ + v = DataHandleUtils.getDoubleRound(2, v.toString().toDouble()) + if(v>=0.3 && v<=20){ return v.toString() } } @@ -644,7 +736,7 @@ class FiledStrand { static def processVirescencePercent(virescence_percent){ if(virescence_percent){ def v = getValue(virescence_percent) - if(v && v>15 && v<=70){ + if(v && v>=10 && v<=70){ return v.toString() } } @@ -788,6 +880,20 @@ class FiledStrand { return null } + static def processUndergroundFloor(floor){ + def v = getValue(floor.toString()); + if(v){ + if(v>=1 && v<=5){ + try { + return v.toString(); + }catch(Exception e){ + println e.getMessage() + } + } + } + return null + } + static def processRealUsePercent(percent){ def v = getValue(percent.toString()); if(v){ @@ -834,15 +940,40 @@ class FiledStrand { static def processCompany(val){ if(val){ - if(val.length()>20){ + if(val.length()>20 || val.length()<4){ return null }else{ - if(val.contains("暂无")){ + if(["#NA","*"].contains(val)){ + return null + } + if(val.contains("暂无") || val.contains("暂定")){ + return null + } + if(val.contains("未知") || val.contains("不祥")){ + return null + } + if(val.contains("待定") || val.contains("待详情")){ + return null + } + if(val.contains("尚未公开")){ + return null + } + if(val.contains("在线咨询")){ + return null + } + + if(val.contains("平米") || val.contains("元平月")){ return null } if(val == "无"){ return null } + if(val == "-"){ + return null + } + if(val == "--"){ + return null + } if(val.isNumber()){ return null } @@ -863,7 +994,7 @@ class FiledStrand { if (pattern.matcher(val).matches()) { return null } - return val + return val.replace("*","").replace("*","") } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy index 4df41ffc4..19934afd6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy @@ -3,7 +3,11 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.ExcelUtil import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.looplineinfo.InsidePolygon +import com.fangjia.script.looplineinfo.InPolygon import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -13,6 +17,7 @@ import groovy.json.JsonSlurper import groovy.io.FileType import org.bson.types.ObjectId +import java.awt.geom.Point2D import java.util.regex.Pattern class AddDevShapeData { @@ -22,22 +27,175 @@ class AddDevShapeData { public static final def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection dev_mapping_CONN = MongoUtils.getCollectionByName2("fangjia", "dev_mapping", MONGO100_URL) + public static final DBCollection administrative_mapping_coll = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + def devAreaMap = [:] + def fullCityMap = CommonDatautils.queryFullNameCity() + def results = new HashSet() + def existList = new HashSet() static main(args) { def task = new AddDevShapeData(); task.run() // task.streetInfo() +// task.addStreetToMapping()//通过乡镇街道的点是否都在开发区围栏中,判断是否将此乡镇加入映射表中 } + def addStreetToMapping(){ + def citys = CommonDatautils.findAllCity() + for(def city : citys){ + def data = fullCityMap.get(city) + def fullCityName = null + if(data){ + fullCityName = data.city + } + def devShapes = findDevShapeByCity(fullCityName) + if(devShapes){ + def streetShapes = findStreetShapeByCity(fullCityName) + addDevStreetToMapping(fullCityName,streetShapes, devShapes) + } + } + + results.each {s-> + println s + } + existList.each{s-> + println s + } + + println " 新增映射乡镇:"+results.size()+" 已存在乡镇:"+existList.size() + + } + + def addDevStreetToMapping(city,streetShapes, devShapes){ + def addCount = 0 + def existsCount = 0 + + + for(def streetObj : streetShapes){ + + def coordinates = streetObj.shape.coordinates + for(def pointList : coordinates){ + def objList = findDevInStreetRange(streetObj, pointList, devShapes) + if(objList){ + for(def obj : objList){ + def existsFlag = isExistStreetByDevName(obj.name, streetObj.street) + if(existsFlag){ + existsCount++ + existList << "[存在] "+obj.name+" "+ streetObj.street + }else{ + addCount++ + results << "[新增] "+obj.name+"#"+ streetObj.street + if(!streetObj.street){ + println streetObj + } + } + } + } + } + } + + + } + + def findDevInStreetRange(streetObj,pointList, devShapes){ + def devList = [] + def minDis =200 + for(def devObj : devShapes){ + def flagList = [] + def distances = [] + for(def pointArr : pointList){ +// def flag = InsidePolygon.isInArea(pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble(), InsidePolygon.createPoint4(devObj.shape.coordinates)) + Point2D.Double point = new Point2D.Double(DataHandleUtils.getDoubleRound(6,pointArr[0].toString().toDouble()), DataHandleUtils.getDoubleRound(6, pointArr[1].toString().toDouble())); + def flag = InPolygon.isInArea(point, InPolygon.createPoint(devObj.shape.coordinates)) + flagList << flag + if(!flag){ + distances << findNearestDistances(pointArr, devObj.shape.coordinates) + } + } + + def trueCount = statResult(flagList) + def pre = trueCount*100/flagList.size() + println streetObj.district+" "+streetObj.street +" "+devObj.name +" "+flagList.size()+" "+pre+"%" + if(pre>=100){ + devList << devObj + } +// else{ +// if(distances && pre>=50){ +// def minCount = 0 +// for(def d : distances){ +// if(d <= minDis){ +// minCount++ +// } +// } +// def simlarPointPre = minCount*100/distances.size() +// if(simlarPointPre>=90){ +// devList << devObj +// } +// println "不在多边型范围内的点,距离小于${minDis}米有:"+minCount+" 占比:"+simlarPointPre +// } +// } + } + + return devList + } + + def findNearestDistances(pointArr, coordinates){ + def minDis = Integer.MAX_VALUE + for(def pointlist : coordinates){ + for(def devPoint : pointlist){ + def distance = GeoUtils.calculateDistance(pointArr[0], pointArr[1], devPoint[0], devPoint[1]) + if(distance < minDis){ + minDis = distance + } + } + } + return minDis + } + + def statResult(rows){ + def count = 0 + for(def s : rows){ + if(s){ + count++ + } + } + return count + } + + def findStreetShapeByCity(city){ + def list = [] + def dbcurosr = administrative_mapping_coll.find(new BasicDBObject("city", city).append("type", "乡镇街道").append("shape", new BasicDBObject('$exists', true))) + for(def d : dbcurosr){ + list << d + } + return list + } + def findDevShapeByCity(city){ + def list = [] + def dbcurosr = DEVELOPMENT_ZONE_43_COLL.find(new BasicDBObject("city", city).append('shape', new BasicDBObject('$exists',true))) + for(def d : dbcurosr){ + list << d + } + return list + } + + def isExistStreetByDevName(devName, street){ + def list = dev_mapping_CONN.find(new BasicDBObject("name", devName).append("type","乡镇街道").append("township",street)).toList() + if(list){ + return true + } + } + + def run(){ - def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\111开发区\\' + def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\218\\' def dir = new File(directoryPath) if (dir.isDirectory()) { -// dir.eachFileRecurse(FileType.FILES) { file -> -// process(file.absolutePath) -// } - addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx") + dir.eachFileRecurse(FileType.FILES) { file -> + process(file.absolutePath) + } +// addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx") } else { println "The specified path is not a directory." } @@ -46,7 +204,7 @@ class AddDevShapeData { def addDevInfo(file){ ExcelUtils.multiThreadingModify(file, {d-> // if(!d['I']){ - def name = d['D'] + def name = d['E'] def q = new BasicDBObject() q.put('_id',new ObjectId(d['A'])) def obj = DEVELOPMENT_ZONE_43_COLL.findOne(q) @@ -57,18 +215,28 @@ class AddDevShapeData { if(obj.operate_result){ d['I'] = obj.operate_result } - if(obj.m_date){ - d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd") + if(obj.shape){ + d['T'] = "有" } +// if(obj.m_date){ +// d['P'] = DateUtils.formatDateByPattern(obj.m_date, "yyyy-MM-dd") +// } + +// if(devAreaMap){ +// devAreaMap.each {k,v-> +// if(obj.name && k && obj.name.contains(k)){ +// d['M'] = v.area_info +// d['N'] = v.count +// d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") +// } +// } +// +// }else{ +// println name+" 空值" +// } } // } - devAreaMap.each {k,v-> - if(name.contains(k)){ - d['M'] = v.area_info - d['N'] = v.count - d['O'] = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd") - } - } + }) } @@ -101,7 +269,7 @@ class AddDevShapeData { DEVELOPMENT_ZONE_43_COLL.save(obj) }else{ udata.put("shape", resultMap) - udata.put("m_date", new Date()) +// udata.put("m_date", new Date()) udata.put("bd_shape",bd_resultMap) udata.put("operate_result" , "已处理") udata.put("operator", "外包") @@ -124,11 +292,11 @@ class AddDevShapeData { def coordinates = features[0].geometry.coordinates def index = 0 - println name +" 有 "+coordinates.size()+" 块" + println file.toString()+" "+name +" 有 "+coordinates.size()+" 块" areaMap.count = coordinates.size() def areas = [] - if(name.contains("泉州经济技术开发区")){ +// if(name.contains("江苏常熟综合保税区")){ for(def pointList : coordinates){ index ++ def gaode_coordinates = [] @@ -171,7 +339,7 @@ class AddDevShapeData { } areaMap.area_info = areas.join("\n") - } +// } }else{ println file +" 有多个" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy index 62d1f299e..c7bcaf957 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy @@ -4,43 +4,104 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.regex.Pattern class AddrAlaisProcess { + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206",27020) + def fullCityMap = CommonDatautils.queryFullNameCity() + static main(args) { def task = new AddrAlaisProcess() // task.run() task.nineApi() +// task.createResult() +// task.mergeResult() } + + def mergeResult(){ + def rets = ExcelUtils.read("D:\\数据\\数据建设\\小区\\ai-结果-0221.xlsx",0,1) + def retMap = [:] + for(def r : rets){ + def num = r['B'].toString().toDouble().toInteger() + def list = retMap.get(num) + if(!list){ + list = new HashSet() + retMap.put(num, list) + } + if(r['C']){ + list << r['C'] + } + + } + def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx" + ExcelUtils.modify(fileName,{d-> + def num = d['A'].toString().toDouble().toInteger() + def aiList = retMap.get(num) + if(aiList){ + d['K'] = aiList.join(",") + } + }) + } + + def createResult(){ + def fileName = "D:\\数据\\数据建设\\小区\\小区别名-待定.xlsx" + def results = [] +// results << ["ID", "城市", "地址"] + ExcelUtils.modify(fileName,{d-> + def str = d['C'] + def addressList = str.split(",") + for(def addr : addressList){ + results << [d['A'].toString().toDouble().toInteger(), d['B'], addr] + } + }) + println results.size() + FileUtils.batchCreateFileByCustomSuffix(results, "D:\\数据\\数据建设\\小区\\ai-待支行.bat", null, "utf-8", false, " ",""); + } + + def nineApi(){ def fileName = "/home/goojia/gj/district/addr_alias.xlsx" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-测试.xlsx" + fileName = "D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx" } ExcelUtils.multiThreadingModify(fileName, {d-> - def addrMap = nineAddrSplit(d['B'], d['E']) + def obj = CommonDatautils.findDistrictById(d['A']) +// def addrMap = nineAddrSplit(obj.city, d['B']) def addrList = [] - if(addrMap && addrMap.address){ - d['G'] = addrMap.city - d['H'] = addrMap.address - addrList << addrMap.address - } - if(d['F']){ - def addrs = d['F'].split(",") - def addr_alias = [] - for(def addr : addrs){ - def map = nineAddrSplit(d['B'], addr) - if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){ - if(!addrList.contains(map.address)){ - addrList << map.address - addr_alias << map.address - } - } - } - d['I'] = addr_alias.join(",") - } +// if(addrMap && addrMap.address){ +// d['C'] = addrMap.road +// d['D'] = addrMap.road_num +// } + if(d['C']){ + def data = fullCityMap.get(obj.city) + if(data){ + def roadObj = findBaiduDatas( data.city, d['C']) + if(roadObj){ + d['E'] = "百度存在" + } + } + } +// if(d['F']){ +// def addrs = d['F'].split(",") +// def addr_alias = [] +// for(def addr : addrs){ +// def map = nineAddrSplit(d['B'], addr) +// if(map && map.address && DataHandleUtils.isStandardRoad(map.address)){ +// if(!addrList.contains(map.address)){ +// addrList << map.address +// addr_alias << map.address +// } +// } +// } +// d['I'] = addr_alias.join(",") +// } }) } @@ -67,9 +128,11 @@ class AddrAlaisProcess { } if (jsonObject.get("street") && jsonObject.get("street") != "null") { address = jsonObject.get("street") + map.road = address } if (jsonObject.get("streetNumber") && jsonObject.get("streetNumber") != "null") { address = address + jsonObject.get("streetNumber") + map.road_num = jsonObject.get("streetNumber") } map.address = address } @@ -95,4 +158,13 @@ class AddrAlaisProcess { // ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区") FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.bat", null, "utf-8", false, "#",""); } + + def findBaiduDatas(city, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("name", name) + q.put('std_tag', Pattern.compile('^道路')) + def obj = BAIDU_POI_COLL.findOne(q) + return obj; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy index 9d223e038..9d4bc3bb9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddDistrictBatch.groovy @@ -4,6 +4,7 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -30,7 +31,10 @@ class AutoAddDistrictBatch extends BaseBatch{ def mcaDistMap = CommonDatautils.queryDistNameCityByMapping() static main(args) { - runBatch(new AutoAddDistrictBatch(), args) +// runBatch(new AutoAddDistrictBatch(), args) + + def task = new AutoAddDistrictBatch(); + task.checkMap() } @Override @@ -52,7 +56,6 @@ class AutoAddDistrictBatch extends BaseBatch{ } ExcelUtils.write(fileName, list, "待入库数据") - check(fileName) } @@ -159,10 +162,10 @@ class AutoAddDistrictBatch extends BaseBatch{ } def findGaoDeDatas(city, region, name){ - String sql = "select name,type from amap_poi_detail where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')" + String sql = "select name,type from amap_poi_2024 where cityname='${city}' and adname='${region}' and MATCH(name, '${name}')" def obj = null ElasticUtils.executeSqlQuery(sql,{ d-> - if(d.name == name && d.type.contains("住宅") ){ //&& d.shape + if(d.name == name){ //&& d.shape obj = d } }) @@ -255,8 +258,37 @@ class AutoAddDistrictBatch extends BaseBatch{ q.put("city", city) q.put("district", district) q.put("name", name) - q.put('std_tag', '房地产;住宅区') +// q.put('std_tag', '房地产;住宅区') def obj = BAIDU_POI_COLL.findOne(q) return obj; } + + + def checkMap(){ + ExcelUtils.modify("D:\\数据\\数据建设\\写字楼\\写字楼存在性判断-test.xlsx",{d-> + + def gaode_obj = findGaodeCount(d['B'], d['C'], d['D']) + if(gaode_obj){ + d['H'] = "是" + d['I'] = gaode_obj.type + }else{ + def baidu_obj = findBaiduCount(d['B'], d['C'], d['D']) + if(baidu_obj){ + d['E'] = "是" + d['F'] = baidu_obj.std_tag + } + } + +// def baidu_obj1 = findBaiduCount(d['B'], d['G'], d['H']) +// if(baidu_obj1){ +// d['M'] = "是" +// } +// def gaode_obj1 = findGaodeCount(d['B'], d['G'], d['H']) +// if(gaode_obj1){ +// d['N'] = "是" +// } +// def score = CosineSimilarAlgorithm.getSimilarityRatio(d['D'], d['H']); +// d['O'] = score + }) + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy index cfa15156b..38c7d1c16 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy @@ -4,10 +4,13 @@ import com.fangjia.core.RegionData import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CopyOnWriteArraySet import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -16,19 +19,18 @@ import java.util.concurrent.atomic.AtomicLong class AutoAddOfficeBatch extends BaseBatch{ def fullCityMap = CommonDatautils.queryFullNameByCity() - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { - runBatch(new AutoAddOfficeBatch(), args) +// runBatch(new AutoAddOfficeBatch(), args) + + def task = new AutoAddOfficeBatch() + task.repeatOffice() } @Override void process(Object args) { -// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/ -// def newName = "开放广场".replaceAll(pattern, "") -// println newName - def citys = CommonDatautils.findAllCity() def allAddOffices = [] allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"] @@ -39,6 +41,7 @@ class AutoAddOfficeBatch extends BaseBatch{ officeMaps = filterOffice(officeMaps) if(officeMaps){ def offices = matchOffice(officeMaps) + allAddOffices.addAll(offices) } } @@ -171,4 +174,73 @@ class AutoAddOfficeBatch extends BaseBatch{ } return null } + + def repeatOffice(){ + def citys = CommonDatautils.findAllCity() + def allrets = [] + allrets << ["ID","城市","区域","名称","地址","可疑ID","可疑城市","可疑区域","可疑名称","可疑地址","相似度","距离"] + for(def c : citys){ + def offices = CommonDatautils.findAllDistricts(c, "office", ["city","region","name","address", "lat", "lng"]) + def rets = checkRepeatOfficeBatch(offices) + println c +" "+offices.size() + allrets.addAll(rets) + } + ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据") + } + + def checkRepeatOfficeBatch(offices){ + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(offices.size()) + def total = offices.size() + def rets = new CopyOnWriteArrayList() + def sets = new CopyOnWriteArraySet() + for(def i=0;imaxScore){ + maxScore = nameScore+addrScore + maxObj = o2 + } + } + } + def distance = -1000 + if(o1.lat && maxObj && maxObj.lat){ + distance = GeoHelper.GetDistance(o1.lat/10000000000.0, o1.lng/10000000000.0, maxObj.lat/10000000000.0, maxObj.lng/10000000000.0) + } + if(maxObj){ + maxObj.distance = distance + maxObj.score = maxScore + return maxObj + } + return null + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy index 50ec412b8..671266e38 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/DevUpdateTestBatch.groovy @@ -33,8 +33,8 @@ class DevUpdateTestBatch { static main(args) { def task = new DevUpdateTestBatch() def path = "D:\\数据\\数据建设\\开发区\\开发区-园区.xlsx"//开发区-园区.xlsx -// task.addDev(path) - task.addDevAddress(path) + task.addDev(path) +// task.addDevAddress(path) // // task.updateHasGov("新疆维吾尔自治区","喀什地区","麦盖提县", "麦盖提镇", "是") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy index 1c94475bc..01b7a0338 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/MinglujiData.groovy @@ -31,12 +31,12 @@ class MinglujiData { } def run(){ - def prov = "河北省" - def city = "石家庄市" -// exportDatas(city) + def prov = "江苏省" + def city = "无锡" + exportDatas(city) // matchDev(prov, city) // matchCommunity(city) - fiveLevelSplit(city) +// fiveLevelSplit(city) // nineLevelSplit(city) // tempMatchDev(prov) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy index abce57fe0..9608aee85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/RegionStandardBatch.groovy @@ -31,8 +31,8 @@ class RegionStandardBatch extends BaseBatch{ @Override void process(Object args) { -// seaweedRegionStandard() - poiRegionStandard() + seaweedRegionStandard() +// poiRegionStandard() } def seaweedRegionStandard(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy index 4ecd49ed0..f799e6651 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownMiddleProcess.groovy @@ -64,7 +64,7 @@ class TownMiddleProcess { //修改乡镇街道别名 // updateTownAlias() //中间表同步到100 -// synchTown() + synchTown() //添加地级市标志 // addCityFlag() // addPinyin() @@ -95,7 +95,7 @@ class TownMiddleProcess { // addTownCenter()//百度接口添加中心点 // checkCenterInTownRang()//检测乡镇街道中心点是否在围栏中 - addSpecialCityTownAlias() +// addSpecialCityTownAlias() } @@ -451,16 +451,16 @@ class TownMiddleProcess { for(def d : dbcursor){ datas << d if (datas.size() >= 2000) { -// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); - ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); +// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) println datas.size() datas.clear(); } } if (datas.size() >= 0) { -// ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); - ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); +// ADMINISTRATIVE_MAPPING_43base_COLL.insert(datas) datas.clear(); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy index 69c368290..0d458ec18 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownStandardBatch.groovy @@ -1,15 +1,21 @@ package com.fangjia.script.gj.data_handle +import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.BaiduMapUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject +import net.sf.json.JSONObject import org.apache.commons.io.FileUtils +import org.bson.types.ObjectId import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse import org.elasticsearch.action.index.IndexRequest @@ -52,14 +58,17 @@ class TownStandardBatch extends BaseBatch{ } def TownStandard(){ -// townStandardByCat("office") + townStandardByCat("office") // townStandardByPoi() - addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 +// addTownByTownShape()//为没有乡镇街道的poi,补充乡镇街道,忽略区域,并使用地图 +// standardTown() // updateTownData() + + } def townStandardByCat(def cat){ //"三明","龙岩","南平","漳州","福州","厦门","泉州","莆田","宁德", - def citys = ["重庆","龙岩","南平","漳州","福州","厦门","泉州","莆田","三明","宁德"]//CommonDatautils.findAllCity() + def citys = ["天津"]//CommonDatautils.findAllCity() for(def city : citys){ def data = fullCityMap.get(city) @@ -70,10 +79,10 @@ class TownStandardBatch extends BaseBatch{ def mzbRegionMap = CommonDatautils.queryMzbRegion(fullCityName) def shortRegionNameMap = CommonDatautils.getRegionShortName(fullCityName) - def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias"], mzbRegionMap) + def townMap = CommonDatautils.queryTownByCity(fullCityName, ["city","district", "street", "alias","shape"], mzbRegionMap) if(townMap){ def regionSet = townMap.keySet() - if(regionSet.size()>1){ + if(regionSet.size()>=1){ haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap) }else{ println city +" 无区域" @@ -139,11 +148,12 @@ class TownStandardBatch extends BaseBatch{ def haveRegionProcess(city, fullCityName, townMap, cat, shortRegionNameMap){ def results = [] if(fullCityName){ - def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township"]) + def districts = CommonDatautils.findAllDistricts(city, cat, ["city","region","name", "township","lng","lat"]) for(def d : districts){ - if(d.township){ - def towns = null - def updateTown = null + + def towns = null + def updateTown = null + if(d.region){ for(def m : townMap){ def fullRegionName = shortRegionNameMap.get(d.region) if(m.key == fullRegionName){ @@ -151,41 +161,51 @@ class TownStandardBatch extends BaseBatch{ break } } + } + if(!towns){ + towns = townMap.get(fullCityName) + } - def opt = true - if(towns){ - for(def town : towns){ - def townAlias = [] - if(town.alias){ - townAlias = town.alias - } - townAlias << town.street - if(townAlias.contains(d.township)){ - if(d.township!=town.street){ - updateTown = town.street - }else{ - opt = false - } - break - } + def opt = true + if(towns){ + for(def town : towns){ + def townAlias = [] + if(town.alias){ + townAlias = town.alias } - } - if(opt){ - if(updateTown){ - def udata = new BasicDBObject() - udata.put("township", updateTown) - udata.put("m_date", new Date()) - println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown -// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + townAlias << town.street + if(townAlias.contains(d.township)){ + if(d.township!=town.street){ + updateTown = town.street + }else{ + opt = false + } + break }else{ - def udata = new BasicDBObject() - udata.put("township", "") - udata.put("m_date", new Date()) - println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值" -// DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + if(town.shape){ + def flag = InsidePolygon.isInArea(d.lng/10000000000.0, d.lat/10000000000.0, InsidePolygon.createPoint4(town.shape.coordinates)) + if(flag){ + updateTown = town.street + break + } + } } } - + } + if(opt){ + if(updateTown){ + def udata = new BasicDBObject() + udata.put("township", updateTown) + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:"+updateTown + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + }else{ + def udata = new BasicDBObject() + udata.put("township", "") + udata.put("m_date", new Date()) + println d.city +" "+d.region+" "+d.name+" "+d.township +" 乡镇修改为:空值" + DbUtils.seaweed.update(new BasicDBObject("_id", d._id), new BasicDBObject('$set', udata)) + } } } } @@ -365,6 +385,33 @@ class TownStandardBatch extends BaseBatch{ return rets; } + def readCSV3(filename){ + def lines = FileUtils.readLines(new File(filename)) + def rets = []; + def index = 0 + for(line in lines) { + if(line && index>=1){ + String[] arrs = line.split(",") + if(arrs.size()>=11 && arrs[10]=="是" && arrs[6]!=arrs[7]){ + def o = new BasicDBObject() + try{ + o.put("id", arrs[0]) + o.put("township", arrs[7]) + o.put("city", arrs[2]) + o.put("old_district", arrs[3]) + o.put("new_district", arrs[8]) + rets << o + }catch(Exception e){ + println arrs + } + } + + } + index++ + } + return rets; + } + def readCSV2(filename){ def lines = FileUtils.readLines(new File(filename)) def rets = []; @@ -380,10 +427,11 @@ class TownStandardBatch extends BaseBatch{ o.put("district", arrs[4]) o.put("lng", arrs[5]) o.put("lat", arrs[6]) - if(arrs.size()<=7){ - o.put("data_source","") - }else{ - o.put("data_source", arrs[7]) + o.put("data_source", arrs[7]) + if(arrs.size()>=9){ + o.put("township", arrs[8]) + o.put("new_district", arrs[9]) + o.put("town_source", arrs[10]) } rets << o }catch(Exception e){ @@ -395,43 +443,13 @@ class TownStandardBatch extends BaseBatch{ return rets; } - def readCSV3(filename){ - def lines = FileUtils.readLines(new File(filename)) - def rets = []; - def index = 0 - for(line in lines) { - if(line && index>=1){ - String[] arrs = line.split(",") - if(arrs.size()>=10 && arrs[7] && arrs[9]=="同区域"){ - def o = new BasicDBObject() - try{ - o.put("id", arrs[0]) - o.put("township", arrs[7]) - if(arrs.size()>=9 && arrs[8]){ - o.put("district", arrs[8]) - } - if(arrs.size()>=10 && arrs[9]){ - o.put("way", arrs[9]) - } - rets << o - }catch(Exception e){ - println arrs - } - } - - } - index++ - } - return rets; - } - def addTownByTownShape(){ def i = 0 def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"] - while(i<50){ + while(i<=50){ def fileName = "/home/goojia/gj/new_poi/input/map/no_township_${i}.csv" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\原始\\no_township_${i}.csv" + fileName = "D:\\数据\\数据建设\\乡镇街道\\no_township_${i}.csv" } File file=new File(fileName); if(file.exists()) { @@ -451,19 +469,53 @@ class TownStandardBatch extends BaseBatch{ } } + + def standardTown(){ + def i = 49 + def headers = ["_id","province","city","district","lng","lat","data_source","乡镇街道", "新区域", "方式", "标准乡镇"] + + def cityTownMap = CommonDatautils.queryAllTown(["city", "district", "mzb_district_name", "street", "alias"]) + while(i<=50){ + def fileName = "/home/goojia/gj/new_poi/input/map/no_township_result_${i}.csv" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图\\no_township_result_${i}.csv" + } + File file=new File(fileName); + if(file.exists()) { + def rets = readCSV2(fileName) + println i +" "+ rets.size() + def rows = addTownshipBatch2(rets, cityTownMap) + if (CommonDatautils.isWindow()) { + com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv", headers, "utf-8", false, ',', ''); + }else{ + com.fangjia.script.gj.ccb.FileUtils.batchCreateFileByCustomSuffix(rows, "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv", headers, "utf-8", false, ',', ''); + } + println fileName +" 运行完成" + }else{ + println fileName +" 不存在" + } + i++ + } + } + def addTownshipBatch(def rets){ def results = new CopyOnWriteArrayList() def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(rets.size()) for(def i=0;i= 1) { @@ -476,6 +528,46 @@ class TownStandardBatch extends BaseBatch{ } finally { latch.countDown() } +// } +// }); + } + latch.await() + return results + } + + + def addTownshipBatch2(def rets, def cityTownMap){ + def results = new CopyOnWriteArrayList() + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + for(def i=0;i= 1) { + if(townMap.size()>=1){ + r.strand_town = isStandTown(townMap, r.new_district, r.township) + } + results << [r.id, r.province, r.city, r.district, r.lng, r.lat, r.data_source, r.township, r.new_district, r.town_source, r.strand_town] + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } } }); } @@ -610,20 +702,21 @@ class TownStandardBatch extends BaseBatch{ } def updateTownData(){ - def i = 0 + def i = 49 while(i<50) { def fileName = "/home/goojia/gj/new_poi/output/no_township_result_${i}.csv" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果\\no_township_result_${i}.csv" + fileName = "D:\\数据\\数据建设\\乡镇街道\\待补充乡镇街道\\结果-地图-验证乡镇\\no_township_result_${i}.csv" } File file = new File(fileName); if (file.exists()) { def rets = readCSV3(fileName) println file.getAbsoluteFile().getName() +" "+(rets.size()*100/200000)+"%"+" "+rets.size() -// if(rets){ -// updateToEs(rets) -// } - + if(rets){ +// updateToEs(rets)//不能用这种方法修改,否则其它字段都将丢失 +// updateToEs2(rets) + updateDistTownship(rets) + } } i++ } @@ -677,4 +770,72 @@ class TownStandardBatch extends BaseBatch{ println "结束时间 "+new Date() } + def updateToEs2(rows) { + for (int i = 0; i < rows.size(); i++) { + def map = rows[i] + updateMapPoi(map.id, map.township) + } + } + + + def updateMapPoi(def id, def township){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.89:9200/map_poi_2024/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.township='${township}'", + "lang": "painless" + }, + "query": { + "term": { + "_id": "${id}" + } + } + } + """ + HttpRequest request = new HttpRequest() + try{ + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = JSONObject.fromObject(res) + }catch(Exception e){ + println "error : "+e.getMessage() + } + + } + + def updateDistTownship(rets){ + for(def m : rets){ + + def obj = CommonDatautils.findDistrictById(m.id) + if(obj.township != m.township){ + def udata = new BasicDBObject() + udata.put("township", m.township) + if(!m.new_district.contains(m.old_district)){ + def lists = CommonDatautils.findAllDistrictByName(obj.city, null, obj.name, obj.cat) + if(lists.size()<2){ + continue + } + def region = RegionData.formatRegion(m.city, m.new_district) + if(region){ + def dbo = CommonDatautils.findDistrict(m.city, region, obj.name, obj.cat) + if(dbo){ + udata.put("status", 1) + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata)) + println "删除 "+m.city +" "+obj.name+" "+m.id+" "+obj.cat + continue + }else{ + udata.put("region", region) + } + } + } + udata.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(m.id)), new BasicDBObject('$set', udata)) + } + } + + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy index bbe074507..da5899b14 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/TownshipStrandUpdate.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData +import com.fangjia.core.util.HttpRequest import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.looplineinfo.InsidePolygon import com.fangjia.script.util.CosineSimilarAlgorithm @@ -11,6 +12,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.impl.AMQImpl +import net.sf.json.JSONObject import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -38,8 +40,8 @@ class TownshipStrandUpdate { static main(args) { def task = new TownshipStrandUpdate() - task.process() - task.updateTown() +// task.process() +// task.updateTown() // task.stat() // task.updateDiffRegionTown() @@ -49,6 +51,9 @@ class TownshipStrandUpdate { // task.updateDevMapping() // println new Date().getTime() // task.updateSpecCityTown() + + + task.updateTownshipAlias() } @@ -629,4 +634,69 @@ class TownshipStrandUpdate { } } + + + def updateTownshipAlias(){ + def fileName = "D:\\数据\\数据建设\\乡镇街道\\20250219--乡镇街道添加别名.xlsx" + ExcelUtils.modify(fileName,{d-> + def q = new BasicDBObject("type","乡镇街道").append("city", d["A"]).append("district",d["B"]).append("street",d["C"]) + def list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList() + def newAlias = d['D'] + if(list.size()==0){ + list = q.put("street", d['D']) + newAlias = d['C'] + list = ADMINISTRATIVE_MAPPING_COLL.find(q).toList() + } + if(list && list.size()==1){ + def obj = list[0] + def alias = obj.alias + if(!alias){ + alias = [] + } + if(!alias.contains(newAlias)){ + alias << newAlias + } + if(alias){ + def udata = new BasicDBObject() + udata.put("alias", alias) + udata.put("m_date", new Date()) + udata.put("operator","陈杰") + ADMINISTRATIVE_MAPPING_COLL.update(new BasicDBObject("_id",obj._id), new BasicDBObject('$set',udata)) + } + +// updateStreetAlias(obj._id.toString(), alias.join("','")) + }else{ + println d["C"] +" "+ d["D"] + } + }) + } + + + def updateStreetAlias(def id, def alias){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.89:9200/administrative_mapping/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.alias=['${alias}']", + "lang": "painless" + }, + "query": { + "term": { + "_id": "${id}" + } + } + } + """ + HttpRequest request = new HttpRequest() + try{ + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = JSONObject.fromObject(res) + }catch(Exception e){ + println "error : "+e.getMessage() + } + + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index 35179b3d9..b69e526a4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -336,7 +336,7 @@ class FangjiaClientUtils { try { address = address.replace("{","") // def url = "http://192.168.0.19:31080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") - def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8") + def url = "http://192.168.0.93:8080/FangJiaNlpService/formatFivelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=true&addressAssessment=false" if(city){ url = url +"&city="+URLEncoder.encode(city,"UTF-8") } @@ -354,7 +354,7 @@ class FangjiaClientUtils { def jsonObj = null def content = null def url = null - address = address.replace("{","") + address = address.toString().replace("{","") int retryCount = 0 while (retryCount < 3) { url = "http://192.168.0.93:8080/FangJiaNlpService/formatNinelevel?address="+URLEncoder.encode(address,"UTF-8")+"&complete=false&useMap=false&addressAssessment=true&fields=dev_confi" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy index f17e815dc..1be279a43 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/OfficeFieldCleanBatch.groovy @@ -74,11 +74,11 @@ class OfficeFieldCleanBatch extends BaseBatch{ // addBlock() // addFieldValue() -// updateErrorField() + updateErrorField() // export() // test() // addOffice() - fileAddBlock() +// fileAddBlock() } def export(){ @@ -134,33 +134,196 @@ class OfficeFieldCleanBatch extends BaseBatch{ } def updateErrorField(){ -// def offices = CommonDatautils.findAllDistrict("office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"]) - def offices = CommonDatautils.findAllDistricts("上海","office", ["city","estate_charge","addr_alias","property_life", "total_floor","finish_building_date","real_use_percent", "land_car_site", "underground_car_site","parking_fee"]) + def offices = CommonDatautils.findAllDistrict("office", ["underground_car_site"]) +// def offices = CommonDatautils.findAllDistricts("上海","office", ["build_area"]) def totalCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(offices.size()) for(def i=0;i60 || val < 0.35){ -//// println val -// udata.put("estate_charge", null) -//// } -// } + if(udata){ udata.put("m_date", new Date()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy index e07ac0739..1b4bd8a80 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsMonthHouseStat.groovy @@ -27,7 +27,7 @@ class EsMonthHouseStat { // task.statDistHouse() // task.statRent() - task.statOfficeHouse() +// task.statOfficeHouse() } def statRent(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy index 5cd666999..2e27e41cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/poc/AddPriceTest.groovy @@ -40,16 +40,58 @@ class AddPriceTest { def run(){ def filePath = "D:\\数据\\价格\\惠州.xlsx" // addGXDPrice(filePath) - addCcbPrice(filePath) +// addCcbPrice(filePath) // addThirdPrice(filePath) // addFjwPrice(filePath) // exportPkPrice() // addRegion() // addTradePrice(filePath) - + addThirdSamplePrice("D:\\数据\\建行\\房产估值\\指导价\\级联、国信达价格导入_0313.xlsx") } + def addThirdSamplePrice(filePath){ + ExcelUtils.modify(filePath,{d-> + def dist = CommonDatautils.findDistrictById(d['A']) + def o = new BasicDBObject() + o.put("city" , dist.city) + o.put("region" , dist.region) + o.put("district_name" , dist.name) + o.put("isGuidancePrice" , 0) + o.put("category", "district") + o.put("c_date" , new Date()) + o.put("user_name" , "房价网") + + if(d['B']){ + o.put("price", d['B'].toString().toDouble().toInteger()) + o.put("source", "JL") + if(o.containsKey("_id")){ + o.removeField("_id") + } + confirm_price_date.save(o) + } + if(d['C']){ + o.put("price", d['C'].toString().toDouble().toInteger()) + o.put("source", "GXD") + if(o.containsKey("_id")){ + o.removeField("_id") + } + confirm_price_date.save(o) + } + + if(d['D']){ + o.put("price", d['D'].toString().toDouble().toInteger()) + o.put("source", "FXT") + if(o.containsKey("_id")){ + o.removeField("_id") + } + confirm_price_date.save(o) + } + }) + } + + + def addTradePrice(filePath){ ExcelUtils.multiThreadingModify(filePath, { d -> if(d['Y']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy index 980d2863d..8aa7b2d42 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/village/govLocation.groovy @@ -19,7 +19,7 @@ DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "commun //def s = getTownshipInfo("118.702988,39.740979") def filePath = "D:\\数据\\涉农\\县政府(最终).xlsx" ExcelUtils.modify(filePath, { row -> - if(!row['I'] && row['I']=='宝应县'){ + if(!row['I'] && row['I']=='建瓯市'){ def rets = EsSearchUtils.findPois(row['B'], row['C'], row['C']+"人民政府") if(rets){ def obj = findOneByName(rets, row['C']+"人民政府") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 41ccc9e38..b5f10fe8d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -1,18 +1,23 @@ package com.fangjia.script.gj.xiamen import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.ProvinceData import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.AMQP +import net.sf.json.JSONObject import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch @@ -23,7 +28,7 @@ import com.fangjia.script.common.DataHandleUtils class SeaweedDataSynch extends BaseBatch{ - static def targetUrl = "http://192.168.0.12:9200/seaweed/" + static def targetUrl = "http://192.168.0.89:9200/seaweed/" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; @@ -132,17 +137,44 @@ class SeaweedDataSynch extends BaseBatch{ mcaDistMap = CommonDatautils.queryDistNameCityByMapping() // priceMap = findAllDistrictSellPrice() - def citys = CommonDatautils.findAllCity().reverse() + def citys = ["潮州"]//CommonDatautils.findAllCity() def index = 0 for(def c : citys){ index++ - updating(c, "district") + + def dists = updating(c, "district") + updateEs(c, dists) + + updating(c, "office") updating(c, "shop") + println index +" "+ c + " 完成" } } + def updateEs(city, dists){ + def data = fullCityMap.get(city) + def distEsDatas = findEsSeaweedDatas(data.city) + def resultMap = [:] + for(def d : dists){ + def esObj = distEsDatas.get(d._id.toString()) + def map = [:] + d.each {k,v-> + if(k!="_id"){ + def esVal = esObj.get(k) + if(v!=esVal){ + map.put(k, v) + } + } + } + if(map){ + resultMap.put(d._id.toString(), map) + } + } + return resultMap + } + def updating(city, cat){ def dists = CommonDatautils.findAllDistrictsFromMg(city, cat, ["city","region","name","address","block","mapping_alias","addr_alias","cat","estate_type2","status","township","lng","lat","build_type","household_count","committee","estate_charge","car_site_count","property_life","house_property","build_area","all_area","build_structure","developer","estate_company","cubage_percent","virescence_percent","loudong_count","schools","main_type","main_type_area","create_year"]) def rets = new CopyOnWriteArrayList() @@ -154,15 +186,13 @@ class SeaweedDataSynch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - def map = createData(dist) - //往100 上同步 updateData(map) - - //往12 es上同步 -// def newUrl = targetUrl +"_doc/"+map.id -// CommonDatautils.httpSaveData(newUrl, map) + rets << map +// //往89 es上同步 +// def newUrl = targetUrl +"_doc/"+map._id +// CommonDatautils.httpSaveData89(newUrl, map) println city +" "+cat +" "+totalCount.get() +"/"+dists.size()+" "+dist.name+" "+map._id } catch (Exception e) { @@ -178,6 +208,33 @@ class SeaweedDataSynch extends BaseBatch{ return rets; } + def updateMapPoi(def id, def township){ + def header = [:] + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + String seawaterUrl = "http://192.168.0.89:9200/seaweed/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" + def body = """ + { + "script": { + "source": "ctx._source.township='${township}'", + "lang": "painless" + }, + "query": { + "term": { + "_id": "${id}" + } + } + } + """ + HttpRequest request = new HttpRequest() + try{ + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + def response = JSONObject.fromObject(res) + }catch(Exception e){ + println "error : "+e.getMessage() + } + + } + def updateData(map){ def udata = new BasicDBObject() map.each{k,v-> @@ -196,14 +253,14 @@ class SeaweedDataSynch extends BaseBatch{ if(province){ map.province = ProvinceData.getFullProvince(province) } - map.fangjia_province = province +// map.fangjia_province = province def data = fullCityMap.get(dist.city) if(data){ map.city = data.city area_code = data.mca_city_code } - map.fangjia_city = dist.city +// map.fangjia_city = dist.city def distData = mcaDistMap.get(dist.city+"#"+dist.region) if(distData){ @@ -213,9 +270,8 @@ class SeaweedDataSynch extends BaseBatch{ // map.region = dist.region println dist.city +" "+dist.region+" 民政部中没有找到相应的区域全称" } - map.area_code = area_code - - map.fangjia_region = dist.region +// map.area_code = area_code +// map.fangjia_region = dist.region map.name = dist.name @@ -390,6 +446,14 @@ class SeaweedDataSynch extends BaseBatch{ DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(row['A'])), new BasicDBObject('$set', udata)) } }) + } + def findEsSeaweedDatas(city){ + def map = new HashMap() + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) + ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc-> + map.put(id, doc) + }) + return map; } } From dfc885ce3810b3cc624622ec49809951fe79d4f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 3 Apr 2025 17:34:22 +0800 Subject: [PATCH 02/13] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=9D=87=E4=BB=B7?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/PropertyController.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 7c4c4ab00..57f2da122 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -2443,7 +2443,11 @@ public class PropertyController { item.put("city", districtStat.getCity()); item.put("district", districtStat.getRegion()); item.put("name", districtStat.getDistrictName()); - item.put("avgPrice", districtStat.getAvg_price_2b()); + if (districtStat.getAvg_price_2b() != null) { + item.put("avgPrice", districtStat.getAvg_price_2b()); + } else { + item.put("avgPrice", districtStat.getAvgPrice()); + } if ("monthly".equals(model.getPeriodType())) { item.put("date", DateUtil.formatByPattern(districtStat.getWeekend(), "yyyy-MM")); } else { From a6ec66ceea0ec0fed77f228211fe3841bff45cf2 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:43:15 +0800 Subject: [PATCH 03/13] =?UTF-8?q?es=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/util/ElasticUtils.groovy | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 6aba12207..a611fedc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.util +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import net.sf.json.JSONObject import org.apache.http.HttpHost import org.apache.http.auth.AuthScope import org.apache.http.auth.UsernamePasswordCredentials @@ -17,11 +20,13 @@ import org.elasticsearch.action.search.ClearScrollRequest import org.elasticsearch.action.search.SearchRequest import org.elasticsearch.action.search.SearchResponse import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.action.support.WriteRequest import org.elasticsearch.client.RequestOptions import org.elasticsearch.client.RestClient import org.elasticsearch.client.RestClientBuilder import org.elasticsearch.client.RestHighLevelClient import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.search.Scroll import org.elasticsearch.search.SearchHit @@ -262,4 +267,68 @@ class ElasticUtils { client.close() return results; } + + static def modify(id, map, host, indexName){ + BulkRequest request = new BulkRequest(); + int processedCount = 0; + def client = null + if(host && host == "192.168.0.89"){ + client = ElasticUtils89.getClient(host) + }else if(host && host == "192.168.0.52"){ + client = ElasticUtils.getClient(host) + } + + request.add(new IndexRequest(indexName, "_doc", id).source(map, XContentType.JSON)); + if (request.numberOfActions() > 100) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents" + } + } + } + + static def delete(id, host, indexName){ + def header = [:] + if(host && host == "192.168.0.89"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + }else if(host && host == "192.168.0.52"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + } + String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query" + def body = """ + { + "query": { + "bool": { + "must": [ + { + "match": { + "id": "${id}" + } + } + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println(res) + } + } From d9a4f8de07bd1f6b1058e7280915eb74e2ff4f0f Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:43:51 +0800 Subject: [PATCH 04/13] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=BC=80=E5=8F=91?= =?UTF-8?q?=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/SynchDevDataBatch.groovy | 119 ++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy new file mode 100644 index 000000000..dc96e2246 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy @@ -0,0 +1,119 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +import java.util.concurrent.ConcurrentHashMap + +/** + * 将43上的开发区同步到mongo-100, es-89, es-52 + */ +class SynchDevDataBatch extends BaseBatch{ + + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection development_zone_CONN = MongoUtils.getCollectionByName2("fangjia", "development_zone", MONGO100_URL) + + DBCollection DEVELOPMENT_ZONE_43_COLL = MongoUtils.getCollectionByName("fangjia_base", "development_zone", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new SynchDevDataBatch(), args); + } + + @Override + public void process(Object args) { + def devMap = synchDevTo100() +// synchDevToEs(devMap) +// synchDevEs(devMap, "192.168.0.89") + synchDevEs(devMap, "192.168.0.52") + } + + def synchDevTo100(){ + def dbcursor = DEVELOPMENT_ZONE_43_COLL.find() + def rets = [] + def devMap = [:] + for(def r : dbcursor){ + rets << r + devMap.put(r._id.toString(), r) + } + //删除旧的,再同步新的 + development_zone_CONN.remove(new BasicDBObject()) + development_zone_CONN.insert(rets) + return devMap; + } + + def synchDevToEs(devMap){ +// def esDevs = findEsDevDatas() +// def ids = [] +// for(def dev : devDatas){ +// ids << dev._id.toString() +// def esObj = esDevs.get(dev._id.toString()) +// def updateValMap = [:] +// if(esObj){ +// dev.each{k,v-> +// if(k!="id" && k!="_id"){ +// def val = dev.get(k) +// if(val!=v){ +// updateValMap.put(k,val) +// } +// } +// } +// } +// if(!esObj || (esObj && updateValMap)){ +// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.89", "development_zone") +// ElasticUtils.modify(dev._id.toString(), dev, "192.168.0.52", "development_zone") +// } +// } +// +// def esDelIds = [] +// esDevs.each{id,map-> +// if(!ids.contains(id)){ +// ElasticUtils.delete(id, "192.168.0.89", "development_zone") +// ElasticUtils.delete(id, "192.168.0.52", "development_zone") +// esDelIds << id +// } +// } +// println "Es删除开发区数量:"+esDelIds.size() + } + + + def synchDevEs(devMap, host){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + + ElasticUtils89.executeQuery(host, "development_zone", queryBuilder, { String id, doc-> + def mongoObj = devMap.get(id) + def updateFieldCount = 0 + mongoObj.each{k,v-> + if(!["id","_id" ,"build_date" ,"create_time","bd_shape","img_url","operator","operate_result","m_date","township","block"].contains(k)){ + def val = doc.get(k) + if(val!=v){ + println doc.name +" 修改 "+k+" "+ v+" "+id +// updateValMap.put(k,val) + updateFieldCount++ + doc.put(k,v) + } + } + } + if(mongoObj && updateFieldCount){ + //进行修改 + return true + }else{ + if(!mongoObj){ + println "删除:"+id +" "+doc.name + ElasticUtils.delete(id, host, "development_zone") + return false + } + } + }) + } + + + + + +} From 076e72419925a658c4781bf9c636d6c602efb84f Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:45:15 +0800 Subject: [PATCH 05/13] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E5=90=8C=E6=AD=A5100=2089=2052?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/xiamen/SeaweedDataSynch.groovy | 208 +++++++++++++++--- 1 file changed, 182 insertions(+), 26 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index b5f10fe8d..65bbe11f6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.util.GeoUtils import com.fangjia.core.util.HttpRequest import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils + import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils @@ -15,7 +16,19 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.rabbitmq.client.AMQP import net.sf.json.JSONObject + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.* +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + import org.bson.types.ObjectId +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.support.WriteRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.xcontent.XContentType import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders @@ -24,15 +37,17 @@ import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong -import com.fangjia.script.common.DataHandleUtils +/** + * 同步小区,商铺,写字楼到89,52上 + */ class SeaweedDataSynch extends BaseBatch{ static def targetUrl = "http://192.168.0.89:9200/seaweed/" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; - DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed", MONGO100_URL) + DBCollection SEAWEED_CONN = MongoUtils.getCollectionByName2("fangjia", "seaweed_dishu", MONGO100_URL) DBCollection administrative_division_100_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division_2023", MONGO100_URL) DBCollection administrative_division_2022_CONN = MongoUtils.getCollectionByName2("fangjia", "administrative_division", MONGO100_URL) @@ -136,43 +151,103 @@ class SeaweedDataSynch extends BaseBatch{ fullCityMap = CommonDatautils.queryFullNameCity() mcaDistMap = CommonDatautils.queryDistNameCityByMapping() -// priceMap = findAllDistrictSellPrice() - def citys = ["潮州"]//CommonDatautils.findAllCity() + priceMap = findAllDistrictSellPrice() + def citys = ["福州"]//CommonDatautils.findAllCity() def index = 0 for(def c : citys){ index++ def dists = updating(c, "district") - updateEs(c, dists) +// updateEs(c, dists, "district", "192.168.0.89") + updateEs(c, dists, "district", "192.168.0.52") +// def offices = updating(c, "office") +// updateEs(c, offices, "office", "192.168.0.89") +// updateEs(c, offices, "office", "192.168.0.52") - updating(c, "office") - updating(c, "shop") +// def shops = updating(c, "shop") +// updateEs(c, shops, "shop", "192.168.0.89") +// updateEs(c, offices, "shop", "192.168.0.52") println index +" "+ c + " 完成" } } - def updateEs(city, dists){ + def updateEs(city, dists, cat, host){ def data = fullCityMap.get(city) - def distEsDatas = findEsSeaweedDatas(data.city) - def resultMap = [:] - for(def d : dists){ - def esObj = distEsDatas.get(d._id.toString()) - def map = [:] - d.each {k,v-> - if(k!="_id"){ - def esVal = esObj.get(k) - if(v!=esVal){ - map.put(k, v) + def distEsDatas = findEsSeaweedDatas(data.city, cat, host) + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + def updateDatas = new CopyOnWriteArrayList() + def ids = new CopyOnWriteArrayList() + //更新89Es + for(def i=0;i + if(!ids.contains(id)){ + delEsData(id, host, "seaweed") + esDelIds << id } } - return resultMap + println "Es 删除 ${cat}类型:"+esDelIds.size() + //将100 mongodb中非正常小区删除 + def mongoIds = findMongoDbDatas(cat, data.city) + + mongoIds.each{id-> + if(!ids.contains(id)){ + SEAWEED_CONN.remove(new BasicDBObject("_id", new ObjectId(id))) + } + } + + } + + def updatingEs(d, esObj, host){ + def map = [:] + def diffMap = [:] + d.each {k,v-> + if(v && k!="_id"){ + map.put(k, v) + } + if(esObj && k!="_id"){ + def esVal = esObj.get(k) + if(esVal!=v){ + diffMap.put(k,v) + } + } + } + if(!esObj || (esObj && diffMap)){ + map.put("id", d._id.toString()) + return map + } + return null } def updating(city, cat){ @@ -392,7 +467,7 @@ class SeaweedDataSynch extends BaseBatch{ map.avg_price = dataMap.avg_price } } - + map.status = 0 // map.update_date = org.apache.commons.httpclient.util.DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") return map } @@ -448,12 +523,93 @@ class SeaweedDataSynch extends BaseBatch{ }) } - def findEsSeaweedDatas(city){ + def findEsSeaweedDatas(city, cat, host){ def map = new HashMap() - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) - ElasticUtils89.executeQuery("192.168.0.89", "seaweed", queryBuilder, { String id, doc-> + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("category", cat)) + ElasticUtils89.executeQuery(host, "seaweed", queryBuilder, { String id, doc-> map.put(id, doc) }) return map; } + + def updateSeaweedEs(updateDatas, host, indexName){ + BulkRequest request = new BulkRequest(); + int processedCount = 0; + def client = null + if(host && host == "192.168.0.89"){ + client = ElasticUtils89.getClient(host) + }else if(host && host == "192.168.0.52"){ + client = ElasticUtils.getClient(host) + } + + updateDatas.each{map-> + request.add(new IndexRequest(indexName, "_doc", map.id).source(map, XContentType.JSON)); + if (request.numberOfActions() >= 100) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + request = new BulkRequest(); + } + } + + if (request.numberOfActions() > 0) { + request.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); + BulkResponse bulkResponse = client.bulk(request, RequestOptions.DEFAULT); + if (bulkResponse.hasFailures()) { + // 处理错误 + println "Bulk request has failures: " + bulkResponse.buildFailureMessage() + } else { + processedCount += request.numberOfActions(); + println "Processed " + processedCount + " documents "+host + } + } + } + + def delEsData(id, host, indexName){ + def header = [:] + if(host && host == "192.168.0.89"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:elasticsearch".getBytes("UTF-8"))}".toString() + }else if(host && host == "192.168.0.52"){ + header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("elastic:goojia123456".getBytes("UTF-8"))}".toString() + } + String seawaterUrl = "http://${host}:9200/${indexName}/_delete_by_query" + def body = """ + { + "query": { + "bool": { + "must": [ + { + "match": { + "id": "${id}" + } + } + ] + } + } + } + """ + HttpRequest request = new HttpRequest() + String res = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println(res) + } + + def findMongoDbDatas(cat, city){ + def q = new BasicDBObject("category", cat).append("city", city) + def keys = new BasicDBObject() + keys.put("_id",1) + def dbcursor = SEAWEED_CONN.find(q, keys) + def mongoIds = [] + for(def d : dbcursor){ + mongoIds << d._id.toString() + } + return mongoIds + } + + } From 961887256046b35527533450bb7d6376497eaffd Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:45:55 +0800 Subject: [PATCH 06/13] =?UTF-8?q?=E5=BC=80=E5=8F=91=E5=8C=BA=E5=9B=B4?= =?UTF-8?q?=E6=A0=8F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/data_handle/AddDevShapeData.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy index 19934afd6..a2d1c56b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddDevShapeData.groovy @@ -189,13 +189,13 @@ class AddDevShapeData { def run(){ - def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\218\\' + def directoryPath = 'D:\\数据\\数据建设\\开发区外包\\临时-没付费\\' def dir = new File(directoryPath) if (dir.isDirectory()) { dir.eachFileRecurse(FileType.FILES) { file -> process(file.absolutePath) } -// addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx") + addDevInfo("D:\\数据\\数据建设\\开发区外包\\所有开发区.xlsx") } else { println "The specified path is not a directory." } From 780f1dccb135797f93d5560b14d2d98fe1b9507c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:47:28 +0800 Subject: [PATCH 07/13] =?UTF-8?q?=E6=8C=87=E5=AE=9Ahost?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/xiamen/ElasticUtils89.groovy | 41 ++++++++++++++++--- 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy index a401bf480..b1b772b92 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/ElasticUtils89.groovy @@ -99,6 +99,29 @@ class ElasticUtils89 { return new RestHighLevelClient(builder) } + static RestHighLevelClient getClientOther(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));// + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + static RestHighLevelClient getClient89(String host) { RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); @@ -174,7 +197,12 @@ class ElasticUtils89 { static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){ - RestHighLevelClient client = getClient(host) + RestHighLevelClient client = null + if(host == "192.168.0.89"){ + client = getClient(host); + }else if(host == "192.168.0.52"){ + client = getClientOther(host); + } BulkProcessor bulkProcessor = createBulkProcessor(host, client) final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) SearchRequest searchRequest = new SearchRequest(indexName); @@ -190,10 +218,11 @@ class ElasticUtils89 { while (searchHits != null && searchHits.length > 0) { for (int k=0; k < searchHits.size(); k++) { final SearchHit hit = searchHits[k]; - executorService.submit(new Runnable() { - @Override - void run() { +// executorService.submit(new Runnable() { +// @Override +// void run() { try{ + if(closure.getMaximumNumberOfParameters() == 1){ def update = closure.call(hit.getSourceAsMap()) if(update){ @@ -208,8 +237,8 @@ class ElasticUtils89 { }catch (Exception e){ e.printStackTrace() } - } - }) +// } +// }) } SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); scrollRequest.scroll(scroll); From 0a99a465d0d3861e9316775af143060f04d7ca2c Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 14:48:29 +0800 Subject: [PATCH 08/13] =?UTF-8?q?=E5=8C=BA=E5=9F=9F=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/location/RegionPoint.groovy | 97 ++++++++++++++++--- 1 file changed, 82 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy index e80dd9c35..030658e63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/RegionPoint.groovy @@ -5,6 +5,7 @@ import com.fangjia.core.RegionData import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.Bytes import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.math.RandomUtils @@ -17,8 +18,9 @@ import java.util.regex.Pattern */ class RegionPointCrawl { - static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") + static DBCollection POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline_2025") static DBCollection GAODECITYCODECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "gaode_city_code") + static DBCollection OLD_POLYLINECOLL = MongoUtils.getCollectionByAuth20190102("fangjia_crawl", "region_polyline") static main(args) { def task = new RegionPointCrawl(); @@ -26,16 +28,19 @@ class RegionPointCrawl { //4个直辖市 北京市、上海市、天津市、重庆市 //5个自治区 "内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区" // def provinces = ["河北省", "山西省","吉林省","辽宁省", "黑龙江省", "陕西省", "甘肃省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省", "云南省"] - def provinces = ["甘肃省"] - def index = 0 - for(def p : provinces){ - index ++ - println index +"/"+provinces.size() +" "+p - task.crawlCityAndDistrictCode(p); - println "城市 区域 code 完成" - task.run(p); - println "区域围栏 完成" - } + //已跑:甘肃省,河北省,山西省,"吉林省","辽宁省", "黑龙江省", "陕西省", "青海省", "山东省", "福建省", "浙江省", "台湾省", "河南省", "湖北省", "湖南省", "江西省", "江苏省", "安徽省", "广东省", "海南省", "四川省", "贵州省","云南省" +// def provinces = ["北京市","上海市","天津市","重庆市" ] +// def index = 0 +// for(def p : provinces){ +// index ++ +// println index +"/"+provinces.size() +" "+p +// task.crawlCityAndDistrictCode(p); +// println "城市 区域 code 完成" +// task.run(p); +// println "区域围栏 完成" +// } + + task.statSame() } @@ -47,7 +52,10 @@ class RegionPointCrawl { def cityCodes = findCityCode(pro); if(cityCodes){ for(def cityMap : cityCodes){ - def codeMap = findAllDistrictCode(cityMap.city_code); + def codeMap = findAllDistrictCode(cityMap.city_code, "district"); +// if(!codeMap){ +// codeMap = findAllDistrictCode(cityMap.city_code, "street"); +// } def index = 0 for(def m : codeMap){ index ++ @@ -93,7 +101,7 @@ class RegionPointCrawl { def saveGaoDeRegionCode(province, cityName,districts){ if(districts){ for(def dist : districts){ - if(dist.level=="district" || cityName=="东莞市"){ + if(dist.level=="district" || dist.level=="street" ){ saveGaoDeCityRegionCode(province, dist.name, dist.citycode, dist.adcode, dist.center, dist.level); } } @@ -125,10 +133,10 @@ class RegionPointCrawl { return new BasicDBObject("$regex", pattern); } - def findAllDistrictCode(def citycode){ + def findAllDistrictCode(def citycode, def level){ def q = new BasicDBObject() q.put("city_code", citycode) - q.put("level", "district") + q.put("level", level) def dbcursor = GAODECITYCODECOLL.find(q) def rets = [] for(def d : dbcursor){ @@ -149,6 +157,7 @@ class RegionPointCrawl { def o = GAODECITYCODECOLL.findOne(q) if(!o){ + println name +" "+adcode+" 新增" q.put("c_date", new Date()) q.put("center", center) GAODECITYCODECOLL.save(q); @@ -233,4 +242,62 @@ class RegionPointCrawl { def count = RandomUtils.nextInt(KeyList.size()) return KeyList.get(count); } + + def statSame(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject()).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + + def index = 0 + for(def d : dbcursor){ + index++ + def q = new BasicDBObject() + q.put("city",d.city) + q.put("region", d.region) + def obj = OLD_POLYLINECOLL.findOne(q) + if(obj){ + if(obj.polyline!=d.polyline){ + println index +" "+ d.city +" "+d.region+" 2次区域不同" + } + }else{ + println index +" "+ d.city +" "+d.region+" 没有此区域" + } + } + } + + + def regionPolyline(){ + def dbcursor = POLYLINECOLL.find(new BasicDBObject("level","district")) + def map = [:] + for(def d : dbcursor){ + def resultMap = [:] + resultMap.type = "polygon" + def arrs = d.polyline.split("\\|") + def rets = [] + for(def s : arrs){ + def points = s.split(";") + def pointList = [] + for(def pointStr : points){ + def pointArr = pointStr.split(",") + pointList << [pointArr[0].toString().toDouble(), pointArr[1].toString().toDouble()] + } + rets << pointList + } + resultMap.coordinates = rets + if(rets.size()>1) { + resultMap.type = "MultiPolygon" + } + println d.city+" "+d.region+" "+rets.size() + + def udata = new BasicDBObject() + udata.put("shape", resultMap) + def center = d.center + if(center){ + def pointArrs = center.split(",") + udata.put("location", ["lon":pointArrs[0].toString().toDouble(), "lat":pointArrs[1].toString().toDouble()]) + } + + def fullCity = fullCityMap.get(d.city) + map.put(fullCity.city+"|@|"+d.region, udata) + } + return map; + } } From f0548b3fe0dff94fca6cf45cdc7b90c5b5374dda Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 15:15:02 +0800 Subject: [PATCH 09/13] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E5=88=AB=E5=90=8D?= =?UTF-8?q?=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddrAlaisProcess.groovy | 88 ++++++++++++++----- 1 file changed, 64 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy index c7bcaf957..6483d1b7d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy @@ -18,11 +18,45 @@ class AddrAlaisProcess { static main(args) { def task = new AddrAlaisProcess() // task.run() +// task.splitDist() +// task.addAddr() task.nineApi() // task.createResult() // task.mergeResult() } + def splitDist(){ + def rows = ExcelUtils.read("D:\\数据\\数据建设\\小区\\全量小区地址及地址别名.xlsx",0,1) + println "原始总计:"+rows.size() + def rets = [] + for(def d : rows){ + def addr = d['F'] + if(d['G']){ + addr = addr +","+d['G'] + } + + def addrs = addr.split(",") + for(def s : addrs){ + rets << [d['A'], d['B'], d['C'], d['D'], d['E'], s] + } + } + println "总计:"+rets.size() + FileUtils.batchCreateFileByCustomSuffix(rets, "D:\\数据\\数据建设\\小区\\有地址小区地址标准化(一行一个地址).bat", null, "utf-8", false, " ",""); + } + + def addAddr(){ + def fileName = "D:\\数据\\数据建设\\小区\\无地址小区.xlsx" + ExcelUtils.multiThreadingModify(fileName,{d-> + def dist = CommonDatautils.findDistrictById(d['B']) + if(dist){ +// d['F'] = dist.address + if(dist.addr_alias){ + d['G'] = dist.addr_alias.join(",") + } + } + }) + } + def mergeResult(){ def rets = ExcelUtils.read("D:\\数据\\数据建设\\小区\\ai-结果-0221.xlsx",0,1) @@ -66,28 +100,31 @@ class AddrAlaisProcess { def nineApi(){ - def fileName = "/home/goojia/gj/district/addr_alias.xlsx" - if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\数据建设\\写字楼\\写字楼待处理地址.xlsx" - } - ExcelUtils.multiThreadingModify(fileName, {d-> - def obj = CommonDatautils.findDistrictById(d['A']) -// def addrMap = nineAddrSplit(obj.city, d['B']) - def addrList = [] -// if(addrMap && addrMap.address){ -// d['C'] = addrMap.road -// d['D'] = addrMap.road_num -// } - if(d['C']){ - def data = fullCityMap.get(obj.city) - if(data){ - def roadObj = findBaiduDatas( data.city, d['C']) - if(roadObj){ - d['E'] = "百度存在" - } - } - } + def i = 0; + while(i<20) { + def fileName = "/home/goojia/gj/district/dist_addr_${i}.xlsx" + if (CommonDatautils.isWindow()) { + fileName = "D:\\数据\\数据建设\\小区\\dist_addr_0.xlsx" + } + File file=new File(fileName); + if(file.exists()){ + ExcelUtils.multiThreadingModify(fileName, {d-> + def addrMap = nineAddrSplit(d['D'], d['G']) + def addrList = [] + if(addrMap && addrMap.address){ + d['H'] = addrMap.road + d['I'] = addrMap.road_num + } + if(d['H']){ + def data = fullCityMap.get(d['D']) + if(data){ + def roadObj = findBaiduDatas( data.city, d['H']) + if(roadObj){ + d['J'] = "百度存在" + } + } + } // if(d['F']){ // def addrs = d['F'].split(",") // def addr_alias = [] @@ -102,7 +139,10 @@ class AddrAlaisProcess { // } // d['I'] = addr_alias.join(",") // } - }) + }) + println fileName +" 完成 " + } + } } @@ -148,7 +188,7 @@ class AddrAlaisProcess { def num = 0 for(def d : districts){ num++ - def row = [num, d.city, d.region, d.name, d.address] + def row = [num, d._id.toString(), d.city, d.region, d.name, d.address] if(d.addr_alias){ row << d.addr_alias.join(",") } @@ -156,7 +196,7 @@ class AddrAlaisProcess { } } // ExcelUtils.write("D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.xlsx", rows, "小区") - FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化.bat", null, "utf-8", false, "#",""); + FileUtils.batchCreateFileByCustomSuffix(rows, "D:\\数据\\数据建设\\小区\\小区地址及地址别名标准化-20250402.bat", null, "utf-8", false, "#",""); } def findBaiduDatas(city, name){ From d73ee6e1ead0af2ff1e63de53070b38f41632911 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 16:05:51 +0800 Subject: [PATCH 10/13] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...SynchAdministrativeMappingDataBatch.groovy | 83 +++++++++++++++++++ .../gj/data_handle/SynchDevDataBatch.groovy | 3 +- 2 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy new file mode 100644 index 000000000..333ad1ef8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy @@ -0,0 +1,83 @@ +package com.fangjia.script.gj.data_handle + +import com.fangjia.script.BaseBatch +import com.fangjia.script.gj.xiamen.ElasticUtils89 +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class SynchAdministrativeMappingDataBatch extends BaseBatch{ + def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL) + + DBCollection ADMINISTRATIVE_MAPPING_COLL = MongoUtils.getCollectionByName("business", "administrative_mapping", "mongo.db.fangjia.com") + + static main(args) { + runBatch(new SynchAdministrativeMappingDataBatch(), args); + } + + @Override + public void process(Object args) { + def retMap = synchAdminiMappingTo100() + synchToEs(retMap, "192.168.0.89") +// synchToEs(retMap, "192.168.0.52") + } + + def synchAdminiMappingTo100(){ + //删除旧的,再同步新的 + ADMINISTRATIVE_MAPPING_COLL_100.remove(new BasicDBObject()) + + def dbcursor = ADMINISTRATIVE_MAPPING_COLL.find(new BasicDBObject()).batchSize(1000) + def datas = [] + def retMap = [:] + for(def d : dbcursor){ + datas << d + retMap.put(d._id.toString(), d) + if (datas.size() >= 2000) { + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); + println datas.size() + datas.clear(); + } + } + + if (datas.size() >= 0) { + ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); + datas.clear(); + } + + return retMap; + } + + def synchToEs(devMap, host){ + QueryBuilder queryBuilder = QueryBuilders.boolQuery() + + ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc-> + def mongoObj = devMap.get(id) + def updateFieldCount = 0 + mongoObj.each{k,v-> + if(!["id","_id"].contains(k)){ + def val = doc.get(k) + if(val!=v){ + println doc.name +" 修改 "+k+" "+ v+" "+id +// updateValMap.put(k,val) + updateFieldCount++ + doc.put(k,v) + } + } + } + if(mongoObj && updateFieldCount){ + //进行修改 + return true + }else{ + if(!mongoObj){ + println "删除:"+id +" "+doc.name + ElasticUtils.delete(id, host, "administrative_mapping") + return false + } + } + }) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy index dc96e2246..893da6125 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchDevDataBatch.groovy @@ -28,8 +28,7 @@ class SynchDevDataBatch extends BaseBatch{ @Override public void process(Object args) { def devMap = synchDevTo100() -// synchDevToEs(devMap) -// synchDevEs(devMap, "192.168.0.89") + synchDevEs(devMap, "192.168.0.89") synchDevEs(devMap, "192.168.0.52") } From bfc789349bab98c5683c4fdde1763ce9bad3edac Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 16:31:59 +0800 Subject: [PATCH 11/13] =?UTF-8?q?=E8=BF=87=E6=BB=A4=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../data_handle/SynchAdministrativeMappingDataBatch.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy index 333ad1ef8..5b09c47a3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy @@ -38,7 +38,6 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ retMap.put(d._id.toString(), d) if (datas.size() >= 2000) { ADMINISTRATIVE_MAPPING_COLL_100.insert(datas); - println datas.size() datas.clear(); } } @@ -58,10 +57,10 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ def mongoObj = devMap.get(id) def updateFieldCount = 0 mongoObj.each{k,v-> - if(!["id","_id"].contains(k)){ + if(!["id","_id","c_date","m_date","polyline_source","operator"].contains(k)){ def val = doc.get(k) if(val!=v){ - println doc.name +" 修改 "+k+" "+ v+" "+id + println doc.name +" 修改 "+id // updateValMap.put(k,val) updateFieldCount++ doc.put(k,v) From 5d76a0222ae8f77bfb599270e38182d0895fa746 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Tue, 8 Apr 2025 16:36:30 +0800 Subject: [PATCH 12/13] =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../gj/data_handle/AddrAlaisProcess.groovy | 1 + .../script/gj/xiamen/SeaweedDataSynch.groovy | 16 ++++++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy index 6483d1b7d..45d1c598a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AddrAlaisProcess.groovy @@ -142,6 +142,7 @@ class AddrAlaisProcess { }) println fileName +" 完成 " } + i++ } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy index 65bbe11f6..0955b9921 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/xiamen/SeaweedDataSynch.groovy @@ -152,22 +152,22 @@ class SeaweedDataSynch extends BaseBatch{ mcaDistMap = CommonDatautils.queryDistNameCityByMapping() priceMap = findAllDistrictSellPrice() - def citys = ["福州"]//CommonDatautils.findAllCity() + def citys = CommonDatautils.findAllCity() def index = 0 for(def c : citys){ index++ def dists = updating(c, "district") -// updateEs(c, dists, "district", "192.168.0.89") + updateEs(c, dists, "district", "192.168.0.89") updateEs(c, dists, "district", "192.168.0.52") -// def offices = updating(c, "office") -// updateEs(c, offices, "office", "192.168.0.89") -// updateEs(c, offices, "office", "192.168.0.52") + def offices = updating(c, "office") + updateEs(c, offices, "office", "192.168.0.89") + updateEs(c, offices, "office", "192.168.0.52") -// def shops = updating(c, "shop") -// updateEs(c, shops, "shop", "192.168.0.89") -// updateEs(c, offices, "shop", "192.168.0.52") + def shops = updating(c, "shop") + updateEs(c, shops, "shop", "192.168.0.89") + updateEs(c, offices, "shop", "192.168.0.52") println index +" "+ c + " 完成" } From d1a821af0cf1a06ada7d2a854e8ca8db78e8da5d Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Wed, 9 Apr 2025 09:52:54 +0800 Subject: [PATCH 13/13] =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ...SynchAdministrativeMappingDataBatch.groovy | 36 +++++++++++++++---- 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy index 5b09c47a3..00b912a8c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/SynchAdministrativeMappingDataBatch.groovy @@ -9,6 +9,8 @@ import com.mongodb.DBCollection import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders +import java.util.concurrent.CopyOnWriteArrayList + class SynchAdministrativeMappingDataBatch extends BaseBatch{ def MONGO100_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; DBCollection ADMINISTRATIVE_MAPPING_COLL_100 = MongoUtils.getCollectionByName2("fangjia", "administrative_mapping", MONGO100_URL) @@ -50,20 +52,25 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ return retMap; } - def synchToEs(devMap, host){ + def synchToEs(retMap, host){ QueryBuilder queryBuilder = QueryBuilders.boolQuery() + def oldIds = new CopyOnWriteArrayList() + ElasticUtils89.executeQuery(host, "administrative_mapping", queryBuilder, { String id, doc-> - def mongoObj = devMap.get(id) + oldIds << id + def mongoObj = retMap.get(id) def updateFieldCount = 0 mongoObj.each{k,v-> - if(!["id","_id","c_date","m_date","polyline_source","operator"].contains(k)){ + if(!["id","_id","c_date","m_date","polyline_source","operator","city_simple_pinyin","prefecture_level_city"].contains(k)){ def val = doc.get(k) if(val!=v){ - println doc.name +" 修改 "+id // updateValMap.put(k,val) - updateFieldCount++ - doc.put(k,v) + if(v.type!="MultiPolygon"){ + println k +" 修改 "+id + updateFieldCount++ + doc.put(k,v) + } } } } @@ -78,5 +85,22 @@ class SynchAdministrativeMappingDataBatch extends BaseBatch{ } } }) + + + def addEsDatas = [] + retMap.each{id,map-> + if(!oldIds.contains(id)){ + println "需新增:"+id + map.remove("c_date") + map.remove("m_date") + map.remove("polyline_source") + map.remove("operator") + addEsDatas << map + } + } + if(addEsDatas){ + ElasticUtils89.updateEs(addEsDatas, host, "administrative_mapping") + } + } }