From d7fbf345d8355cd521691cdba064577404a04341 Mon Sep 17 00:00:00 2001 From: gaojuan1 Date: Mon, 14 Apr 2025 13:10:09 +0800 Subject: [PATCH] update --- .../gj/data_handle/AutoAddOfficeBatch.groovy | 441 +++++++++++++++--- 1 file changed, 373 insertions(+), 68 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy index 38c7d1c16..5338abf79 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data_handle/AutoAddOfficeBatch.groovy @@ -2,12 +2,18 @@ package com.fangjia.script.gj.data_handle import com.fangjia.core.RegionData import com.fangjia.script.BaseBatch +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.util.CosineSimilarAlgorithm import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CopyOnWriteArraySet @@ -16,30 +22,47 @@ import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong -class AutoAddOfficeBatch extends BaseBatch{ +class AutoAddOfficeBatch extends BaseBatch { - def fullCityMap = CommonDatautils.queryFullNameByCity() - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + def fullCityMap = CommonDatautils.queryFullNameByCity() + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) + DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206", 27020) + + //235为房价网内部计算均价的库 + def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified", + "postgres", "Goojia123456", "org.postgresql.Driver") static main(args) { // runBatch(new AutoAddOfficeBatch(), args) def task = new AutoAddOfficeBatch() - task.repeatOffice() +// task.repeatOffice() + task.repeatDist() } @Override void process(Object args) { +// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/ +// def newName = "开放广场".replaceAll(pattern, "") +// println newName +// findAddOfficeDatas() //查找需要新增的写字楼数据 +// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据 +// addOffice()//新增写字楼 + + + } + + def findAddOfficeDatas() { def citys = CommonDatautils.findAllCity() def allAddOffices = [] - allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"] - for(def c : citys){ + allAddOffices << ["地图ID", "省", "市", "区", "写字楼名称", "地址", "经度", "纬度", "匹配城市", "匹配区域", "匹配名称", "匹配方式"] + for (def c : citys) { def cityObj = fullCityMap.get(c) - if(cityObj){ + if (cityObj) { def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼") officeMaps = filterOffice(officeMaps) - if(officeMaps){ + if (officeMaps) { def offices = matchOffice(officeMaps) allAddOffices.addAll(offices) @@ -49,59 +72,72 @@ class AutoAddOfficeBatch extends BaseBatch{ ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据") } - def filterOffice(officeMaps){ + def checkInBaiduMapOffice() { + ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx", { d -> + if (!d['L']) { + def obj = findBaiduDatas(d['C'], d['D'], d['E']) + if (obj) { + d['L'] = "百度存在" + d['M'] = obj.std_tag.toString() + } + } + + }, 0) + } + + def filterOffice(officeMaps) { def offices = [] def keys = [] - for(def o : officeMaps){ - if(!o){ + for (def o : officeMaps) { + if (!o) { continue } - if(o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")){ - println "[ 过滤 ] "+o.city+" "+o.name + if (o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")) { + println "[ 过滤 ] " + o.city + " " + o.name continue } - if(o.name.contains("路写字楼")){ - println "[ 过滤 ] "+o.city+" "+o.name + if (o.name.contains("路写字楼")) { + println "[ 过滤 ] " + o.city + " " + o.name continue } def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/ def newName = o.name.replaceAll(pattern, "") o.name = newName - def keyName = o.city +"#"+o.district+"#"+o.name - if(!keys.contains(keyName)){ + def keyName = o.city + "#" + o.district + "#" + o.name + if (!keys.contains(keyName)) { offices << o keys << keyName } } - println "原:"+officeMaps.size() +" 新:"+offices.size() + println "原:" + officeMaps.size() + " 新:" + offices.size() return offices } - def matchOffice(offices){ + def matchOffice(offices) { def addCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(offices.size()) def total = offices.size() def rets = new CopyOnWriteArrayList() - for(def i=0;i=1){ + if (rets) { + if (rets.size() >= 1) { office = rets[0] } - }else{ + } else { rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office") - if(rets && rets.size()>=1){ + if (rets && rets.size() >= 1) { office = rets[0] - }else{ + } else { rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district") - if(rets && rets.size()>=1){ + if (rets && rets.size() >= 1) { office = rets[0] } } } - if(office){ + if (office) { office.match_way = "数据库" } return office @@ -144,65 +180,147 @@ class AutoAddOfficeBatch extends BaseBatch{ return null } - def matchApi(nmd){ + def matchApi(nmd) { def address = nmd.city - if(nmd.district){ + if (nmd.district) { address = address + nmd.district } address = address + nmd.name def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false) - if(!jsonObject){ - if(DataUtil.isPreciseAddress(nmd.address)){ + if (!jsonObject) { + if (DataUtil.isPreciseAddress(nmd.address)) { address = nmd.city - if(nmd.district){ + if (nmd.district) { address = address + nmd.district } address = address + nmd.address jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false) } } - if(jsonObject) { - def udata = [:] - udata.put("_id",jsonObject.get("id")) - udata.put("city",jsonObject.get("city")) - udata.put("region",jsonObject.get("district")) - udata.put("name",jsonObject.get("name")) - udata.put("match_type",jsonObject.get("propertyType")) - udata.put("match_credit",jsonObject.get("credit")) + if (jsonObject) { + def udata = [:] + udata.put("_id", jsonObject.get("id")) + udata.put("city", jsonObject.get("city")) + udata.put("region", jsonObject.get("district")) + udata.put("name", jsonObject.get("name")) + udata.put("match_type", jsonObject.get("propertyType")) + udata.put("match_credit", jsonObject.get("credit")) udata.put("match_way", "接口") return udata } return null } - def repeatOffice(){ + def repeatOffice() { def citys = CommonDatautils.findAllCity() def allrets = [] - allrets << ["ID","城市","区域","名称","地址","可疑ID","可疑城市","可疑区域","可疑名称","可疑地址","相似度","距离"] - for(def c : citys){ - def offices = CommonDatautils.findAllDistricts(c, "office", ["city","region","name","address", "lat", "lng"]) + allrets << ["ID", "城市", "区域", "名称", "地址", "可疑ID", "可疑城市", "可疑区域", "可疑名称", "可疑地址", "相似度", "距离"] + for (def c : citys) { + def offices = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "address", "lat", "lng"]) def rets = checkRepeatOfficeBatch(offices) - println c +" "+offices.size() + println c + " " + offices.size() allrets.addAll(rets) } ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据") } - def checkRepeatOfficeBatch(offices){ + def repeatDist(){ + def citys = CommonDatautils.findAllCity() + def allrets = [] + allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"] + for (def c : citys) { + def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"]) + def rets = checkRepeatDistrictBatch(c, districts) + println c + " " + districts.size() + allrets.addAll(rets) + println c +" 支行完成 "+rets.size() + } + ExcelUtils.write("/home/goojia/gj/重复可疑小区1.xlsx", allrets, "数据") + } + + def checkRepeatDistrictBatch(city,districts) { + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def cityId = cityCodeDataMap.get(city) + + def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502) + def addCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(districts.size()) + def total = districts.size() + def rets = new CopyOnWriteArrayList() + def sets = new CopyOnWriteArraySet() + for (def i = 0; i < total; i++) { + final def o1 = districts[i] +// fixedThreadPool.execute(new Runnable() { +// void run() { + try { + addCount.addAndGet(1) +// if(!o1.name.contains("四公司")){ +// continue +// } + if(!sets.contains(o1._id.toString())){ + def similarRets = findListSimilar(o1, districts) + if (similarRets) { + def newPriceMap1 = fjwPriceMap.get(o1._id.toString()) + def new_price1 = null + if(newPriceMap1){ + new_price1 = newPriceMap1.calc_type + } + def ailas_str1 = "" + if(o1.alias){ + ailas_str1 = o1.alias.join(",") + } + def rows = [] + for(def similar : similarRets){ + if (!sets.contains(similar._id.toString())) { + sets.add(similar._id.toString()) + def ailas_str = "" + if(similar.alias){ + ailas_str = similar.alias.join(",") + } + def newPriceMap = fjwPriceMap.get(o1._id.toString()) + def new_price = null + if(newPriceMap){ + new_price = newPriceMap.calc_type + } + rows << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score] +// rets << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score] + } + } + if(rows){ + sets.add(o1._id.toString()) + rets << [ o1.city, o1.region, o1.name, ailas_str1,new_price1, o1.address, "主", "主", "主"] + rets.addAll(rows) + } + + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } +// } +// }); + } +// latch.await() + return rets + } + + def checkRepeatOfficeBatch(offices) { def addCount = new AtomicLong() final CountDownLatch latch = new CountDownLatch(offices.size()) def total = offices.size() def rets = new CopyOnWriteArrayList() def sets = new CopyOnWriteArraySet() - for(def i=0;i o2.name.length()){ + str = o1.name.replace(o2.name, "") + }else{ + str = o2.name.replace(o1.name, "") + } + if (nameScore > 0.6 && ["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) { + o2.nameScore = nameScore + o2.addrScore = addrScore + o2.score = nameScore + rets << o2 + ids << o2._id.toString() + def maxObj = findMaxSimilar(ids, o2, datas) + while(maxObj){ + maxObj.nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(maxObj.name, o1.name); + if(o1.address && maxObj.address){ + maxObj.addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address); + } + maxObj.score = maxObj.nameScore + rets << maxObj + ids << maxObj._id.toString() + maxObj = findMaxSimilar(ids, maxObj, datas) + } + } + } + } + + rets.sort{a,b-> + def a1 = a.score + def b1 = b.score + return b1.compareTo(a1) + } + + if (rets.size()>2) { + return rets[0..2] + } + return rets + } + + def findMaxSimilar(ids, o1, datas){ + def maxObj = null + def maxScore = -1000 + for (def o2 : datas) { + if (o1._id.toString() != o2._id.toString() && o1.region==o2.region && !ids.contains(o2._id.toString())) { + def name1 = o1.name//.replace("小区","").replace("别墅","") + def name2 = o2.name//.replace("小区","").replace("别墅","") + def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2); + def addrScore = null + if(o1.address && o2.address){ + addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address); + } + def str = null + if(o1.name.length() > o2.name.length()){ + str = o1.name.replace(o2.name, "") + }else{ + str = o2.name.replace(o1.name, "") + } + if (nameScore > 0.6 && nameScore>maxScore && ["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) { + maxObj = o2 + o2.nameScore = nameScore + o2.addrScore = addrScore + o2.score = nameScore + } + } + } + + return maxObj + } + + def findOneSimilarOffice(o1, offices) { def maxScore = 0 def maxObj = null - for(def o2 : offices){ - if(o1._id.toString()!=o2._id.toString()){ + for (def o2 : offices) { + if (o1._id.toString() != o2._id.toString()) { double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name); - double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address,o2.address); - if(nameScore+addrScore>maxScore){ - maxScore = nameScore+addrScore + double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address, o2.address); + if (nameScore + addrScore > maxScore) { + maxScore = nameScore + addrScore maxObj = o2 } } } def distance = -1000 - if(o1.lat && maxObj && maxObj.lat){ - distance = GeoHelper.GetDistance(o1.lat/10000000000.0, o1.lng/10000000000.0, maxObj.lat/10000000000.0, maxObj.lng/10000000000.0) + if (o1.lat && maxObj && maxObj.lat) { + distance = GeoHelper.GetDistance(o1.lat / 10000000000.0, o1.lng / 10000000000.0, maxObj.lat / 10000000000.0, maxObj.lng / 10000000000.0) } - if(maxObj){ + if (maxObj) { maxObj.distance = distance maxObj.score = maxScore return maxObj } return null } + + + def findBaiduDatas(city, district, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("district", district) + q.put("name", name) +// q.put('std_tag', '房地产;住宅区') + def obj = BAIDU_POI_COLL.findOne(q) + return obj; + } + + def addOffice(){ + ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d-> + if(d['L']!='人工删除'){ + def city = RegionData.formatCity(d['C']) + def region = RegionData.formatRegion(city, d['D']) + if(city && region){ + def locationstr = d['G'].toString()+","+d['H'].toString() + def bdpoint = FangjiaClientUtils.convert(locationstr, 1) + def udata = new BasicDBObject() + if(bdpoint){ + udata.put("lng2", (bdpoint[0]*10000000000).longValue()) + udata.put("lat2", (bdpoint[1]*10000000000).longValue()) + } + + udata.put("city", city) + udata.put("region", region) + udata.put("name", d['E']) + udata.put("address", d['I']) + udata.put("cat", "office") + udata.put("status", 0) + udata.put("c_date",new Date()) + udata.put("m_date",new Date()) + udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue()) + udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue()) + + def estate_type = null + if(d['M'] && d['M'].contains("园区")){ + estate_type = "创意园区" + } + + if(!estate_type && d['E'].contains("创意")){ + estate_type = "创意园区" + } + if(estate_type){ + udata.put("estate_type2", estate_type) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + udata.put("mark_map", markMap) + DbUtils.seaweed.save(udata) + println d['E'] +" "+udata._id.toString() + } + } + }) + } + + + def findNewFjwAvgPrice(def commid, def cityid){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1" + def map = [:] + pgsqlInstance235.eachRow(sql, { r -> + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + }) + return map + } + + def findNewFjwAvgPriceList(def cityid, date){ + def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr,gujia_bds_id from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujiabds_clc_yrmo_val=${date}" + def retMap = [:] + pgsqlInstance235.eachRow(sql, { r -> + def map = [:] + map.avg_price = r.gujia_bds_clc_avprc + map.type = r.gujia_bds_clmtd + map.month = r.gujiabds_clc_yrmo_val + map.credit_value = r.gujia_bds_cfdc_dgr + if(r.gujia_bds_clmtd == 1){ + map.calc_type = "人工估值" + }else if(r.gujia_bds_clmtd == 2){ + map.calc_type = "样本计算" + }else if(r.gujia_bds_clmtd == 3){ + map.calc_type = "神经网络" + } + retMap.put(r.gujia_bds_id, map) + }) + return retMap + } + }