Merge branch 'script_gj' into 'master'

update

See merge request !1112
This commit is contained in:
2025-04-14 13:10:16 +08:00
@@ -2,12 +2,18 @@ package com.fangjia.script.gj.data_handle
import com.fangjia.core.RegionData
import com.fangjia.script.BaseBatch
import com.fangjia.script.ccbevaluation.CcbCommonUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import groovy.sql.Sql
import java.util.concurrent.CopyOnWriteArrayList
import java.util.concurrent.CopyOnWriteArraySet
@@ -16,30 +22,47 @@ import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class AutoAddOfficeBatch extends BaseBatch{
class AutoAddOfficeBatch extends BaseBatch {
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def fullCityMap = CommonDatautils.queryFullNameByCity()
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
DBCollection BAIDU_POI_COLL = MongoUtils.getCollectionByName("crawl", "poi_wap_spider", "192.168.0.206", 27020)
//235为房价网内部计算均价的库
def pgsqlInstance235 = Sql.newInstance("jdbc:postgresql://192.168.0.235:5433/fangjia?stringtype=unspecified",
"postgres", "Goojia123456", "org.postgresql.Driver")
static main(args) {
// runBatch(new AutoAddOfficeBatch(), args)
def task = new AutoAddOfficeBatch()
task.repeatOffice()
// task.repeatOffice()
task.repeatDist()
}
@Override
void process(Object args) {
// def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
// def newName = "开放广场".replaceAll(pattern, "")
// println newName
// findAddOfficeDatas() //查找需要新增的写字楼数据
// checkInBaiduMapOffice()//查寻在百度地图上有的写字楼数据
// addOffice()//新增写字楼
}
def findAddOfficeDatas() {
def citys = CommonDatautils.findAllCity()
def allAddOffices = []
allAddOffices << ["地图ID","省","市","区","写字楼名称","地址","经度","纬度","匹配城市","匹配区域","匹配名称","匹配方式"]
for(def c : citys){
allAddOffices << ["地图ID", "省", "市", "区", "写字楼名称", "地址", "经度", "纬度", "匹配城市", "匹配区域", "匹配名称", "匹配方式"]
for (def c : citys) {
def cityObj = fullCityMap.get(c)
if(cityObj){
if (cityObj) {
def officeMaps = CommonDatautils.findOfficeByMap(cityObj.city, "商务写字楼")
officeMaps = filterOffice(officeMaps)
if(officeMaps){
if (officeMaps) {
def offices = matchOffice(officeMaps)
allAddOffices.addAll(offices)
@@ -49,59 +72,72 @@ class AutoAddOfficeBatch extends BaseBatch{
ExcelUtils.write("D:\\数据\\高德写字楼.xlsx", allAddOffices, "数据")
}
def filterOffice(officeMaps){
def checkInBaiduMapOffice() {
ExcelUtils.multiThreadingModify("D:\\数据\\高德写字楼1.xlsx", { d ->
if (!d['L']) {
def obj = findBaiduDatas(d['C'], d['D'], d['E'])
if (obj) {
d['L'] = "百度存在"
d['M'] = obj.std_tag.toString()
}
}
}, 0)
}
def filterOffice(officeMaps) {
def offices = []
def keys = []
for(def o : officeMaps){
if(!o){
for (def o : officeMaps) {
if (!o) {
continue
}
if(o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")){
println "[ 过滤 ] "+o.city+" "+o.name
if (o.name.startsWith("写字楼") || o.name.startsWith("综合楼") || o.name.contains("销售") || o.name.startsWith("办公楼") || o.name.startsWith("公务楼") || o.name.startsWith("行政楼") || o.name.startsWith("商务办公楼") || o.name.startsWith("综合办公楼")) {
println "[ 过滤 ] " + o.city + " " + o.name
continue
}
if(o.name.contains("路写字楼")){
println "[ 过滤 ] "+o.city+" "+o.name
if (o.name.contains("路写字楼")) {
println "[ 过滤 ] " + o.city + " " + o.name
continue
}
def pattern = /[A-Z0-9]座|[0-9a-z]栋|[A-Z]栋|·|\(建设中\)|[0-9]号楼|\(装修中\)|[A-Z]楼|[0-9]幢/
def newName = o.name.replaceAll(pattern, "")
o.name = newName
def keyName = o.city +"#"+o.district+"#"+o.name
if(!keys.contains(keyName)){
def keyName = o.city + "#" + o.district + "#" + o.name
if (!keys.contains(keyName)) {
offices << o
keys << keyName
}
}
println "原:"+officeMaps.size() +" 新:"+offices.size()
println "原:" + officeMaps.size() + " 新:" + offices.size()
return offices
}
def matchOffice(offices){
def matchOffice(offices) {
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
for(def i=0;i<offices.size();i++){
for (def i = 0; i < offices.size(); i++) {
final def nmd = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
try {
addCount.addAndGet(1)
def office = matchDB(nmd)
if(!office){
if (!office) {
office = matchApi(nmd)
}
if(office){
if (office) {
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, office.city, office.region, office.name, office.match_way]
println addCount.get()+"/"+total +" [success] "+ office.get("region")+" "+office.get("name")+" "+office.match_way
}else{
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address,nmd.lng, nmd.lat,"","","",""]
println addCount.get()+"/"+total +" [fail] "+nmd.city+" "+nmd.name
println addCount.get() + "/" + total + " [success] " + office.get("region") + " " + office.get("name") + " " + office.match_way
} else {
rets << [nmd.map_id, nmd.province, nmd.city, nmd.district, nmd.name, nmd.address, nmd.lng, nmd.lat, "", "", "", ""]
println addCount.get() + "/" + total + " [fail] " + nmd.city + " " + nmd.name
}
} catch (Exception e) {
e.printStackTrace()
@@ -115,28 +151,28 @@ class AutoAddOfficeBatch extends BaseBatch{
return rets
}
def matchDB(nmd){
def matchDB(nmd) {
def simpleCity = RegionData.formatCity(nmd.city)
def simpleDistrict = RegionData.formatRegion(simpleCity, nmd.district)
if(simpleCity && simpleDistrict){
if (simpleCity && simpleDistrict) {
def office = null
def rets = CommonDatautils.findDistrictList(simpleCity, simpleDistrict, nmd.name, "office")
if(rets){
if(rets.size()>=1){
if (rets) {
if (rets.size() >= 1) {
office = rets[0]
}
}else{
} else {
rets = CommonDatautils.findDistrictByAlias(simpleCity, simpleDistrict, nmd.name, "office")
if(rets && rets.size()>=1){
if (rets && rets.size() >= 1) {
office = rets[0]
}else{
} else {
rets = CommonDatautils.findDistrictByAddress(simpleCity, simpleDistrict, nmd.name, "district")
if(rets && rets.size()>=1){
if (rets && rets.size() >= 1) {
office = rets[0]
}
}
}
if(office){
if (office) {
office.match_way = "数据库"
}
return office
@@ -144,65 +180,147 @@ class AutoAddOfficeBatch extends BaseBatch{
return null
}
def matchApi(nmd){
def matchApi(nmd) {
def address = nmd.city
if(nmd.district){
if (nmd.district) {
address = address + nmd.district
}
address = address + nmd.name
def jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
if(!jsonObject){
if(DataUtil.isPreciseAddress(nmd.address)){
if (!jsonObject) {
if (DataUtil.isPreciseAddress(nmd.address)) {
address = nmd.city
if(nmd.district){
if (nmd.district) {
address = address + nmd.district
}
address = address + nmd.address
jsonObject = FangjiaClientUtils.newMatch(nmd.city, address, "office", false)
}
}
if(jsonObject) {
def udata = [:]
udata.put("_id",jsonObject.get("id"))
udata.put("city",jsonObject.get("city"))
udata.put("region",jsonObject.get("district"))
udata.put("name",jsonObject.get("name"))
udata.put("match_type",jsonObject.get("propertyType"))
udata.put("match_credit",jsonObject.get("credit"))
if (jsonObject) {
def udata = [:]
udata.put("_id", jsonObject.get("id"))
udata.put("city", jsonObject.get("city"))
udata.put("region", jsonObject.get("district"))
udata.put("name", jsonObject.get("name"))
udata.put("match_type", jsonObject.get("propertyType"))
udata.put("match_credit", jsonObject.get("credit"))
udata.put("match_way", "接口")
return udata
}
return null
}
def repeatOffice(){
def repeatOffice() {
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << ["ID","城市","区域","名称","地址","可疑ID","可疑城市","可疑区域","可疑名称","可疑地址","相似度","距离"]
for(def c : citys){
def offices = CommonDatautils.findAllDistricts(c, "office", ["city","region","name","address", "lat", "lng"])
allrets << ["ID", "城市", "区域", "名称", "地址", "可疑ID", "可疑城市", "可疑区域", "可疑名称", "可疑地址", "相似度", "距离"]
for (def c : citys) {
def offices = CommonDatautils.findAllDistricts(c, "office", ["city", "region", "name", "address", "lat", "lng"])
def rets = checkRepeatOfficeBatch(offices)
println c +" "+offices.size()
println c + " " + offices.size()
allrets.addAll(rets)
}
ExcelUtils.write("D:\\数据\\写字楼\\重复可疑写字楼1.xlsx", allrets, "数据")
}
def checkRepeatOfficeBatch(offices){
def repeatDist(){
def citys = CommonDatautils.findAllCity()
def allrets = []
allrets << [ "城市", "区域", "名称", "别名","计算方式","地址","名称相似度","地址相似度","总相似度"]
for (def c : citys) {
def districts = CommonDatautils.findAllDistricts(c, "district", ["city", "region", "name", "address", "alias"])
def rets = checkRepeatDistrictBatch(c, districts)
println c + " " + districts.size()
allrets.addAll(rets)
println c +" 支行完成 "+rets.size()
}
ExcelUtils.write("/home/goojia/gj/重复可疑小区1.xlsx", allrets, "数据")
}
def checkRepeatDistrictBatch(city,districts) {
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def cityId = cityCodeDataMap.get(city)
def fjwPriceMap = findNewFjwAvgPriceList(cityId, 202502)
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(districts.size())
def total = districts.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for (def i = 0; i < total; i++) {
final def o1 = districts[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
try {
addCount.addAndGet(1)
// if(!o1.name.contains("四公司")){
// continue
// }
if(!sets.contains(o1._id.toString())){
def similarRets = findListSimilar(o1, districts)
if (similarRets) {
def newPriceMap1 = fjwPriceMap.get(o1._id.toString())
def new_price1 = null
if(newPriceMap1){
new_price1 = newPriceMap1.calc_type
}
def ailas_str1 = ""
if(o1.alias){
ailas_str1 = o1.alias.join(",")
}
def rows = []
for(def similar : similarRets){
if (!sets.contains(similar._id.toString())) {
sets.add(similar._id.toString())
def ailas_str = ""
if(similar.alias){
ailas_str = similar.alias.join(",")
}
def newPriceMap = fjwPriceMap.get(o1._id.toString())
def new_price = null
if(newPriceMap){
new_price = newPriceMap.calc_type
}
rows << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
// rets << [ similar.city, similar.region, similar.name, ailas_str,new_price, similar.address, similar.nameScore, similar.addrScore, similar.score]
}
}
if(rows){
sets.add(o1._id.toString())
rets << [ o1.city, o1.region, o1.name, ailas_str1,new_price1, o1.address, "主", "主", "主"]
rets.addAll(rows)
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
// }
// });
}
// latch.await()
return rets
}
def checkRepeatOfficeBatch(offices) {
def addCount = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(offices.size())
def total = offices.size()
def rets = new CopyOnWriteArrayList()
def sets = new CopyOnWriteArraySet()
for(def i=0;i<total;i++){
for (def i = 0; i < total; i++) {
final def o1 = offices[i]
fixedThreadPool.execute(new Runnable() {
void run() {
try{
try {
addCount.addAndGet(1)
def maxObj = findOneSimilarOffice(o1, offices)
if(maxObj){
if(!sets.contains(o1._id.toString())){
if (maxObj) {
if (!sets.contains(o1._id.toString())) {
sets.add(maxObj._id.toString())
rets << [o1._id.toString(), o1.city, o1.region, o1.name, o1.address, maxObj._id.toString(), maxObj.city, maxObj.region, maxObj.name, maxObj.address, maxObj.score, maxObj.distance]
}
@@ -219,28 +337,215 @@ class AutoAddOfficeBatch extends BaseBatch{
return rets
}
def findOneSimilarOffice(o1, offices){
def findListSimilar(o1, datas){
def rets = new CopyOnWriteArrayList()
def ids = []
ids << o1._id.toString()
for (def o2 : datas) {
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region) {
// if(!o2.name.contains("四公司")){
// continue
// }
def name1 = o1.name//.replace("小区","").replace("别墅","")
def name2 = o2.name//.replace("小区","").replace("别墅","")
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
def addrScore = null
if(o1.address && o2.address){
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
}
def str = null
if(o1.name.length() > o2.name.length()){
str = o1.name.replace(o2.name, "")
}else{
str = o2.name.replace(o1.name, "")
}
if (nameScore > 0.6 && ["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
o2.nameScore = nameScore
o2.addrScore = addrScore
o2.score = nameScore
rets << o2
ids << o2._id.toString()
def maxObj = findMaxSimilar(ids, o2, datas)
while(maxObj){
maxObj.nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(maxObj.name, o1.name);
if(o1.address && maxObj.address){
maxObj.addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
}
maxObj.score = maxObj.nameScore
rets << maxObj
ids << maxObj._id.toString()
maxObj = findMaxSimilar(ids, maxObj, datas)
}
}
}
}
rets.sort{a,b->
def a1 = a.score
def b1 = b.score
return b1.compareTo(a1)
}
if (rets.size()>2) {
return rets[0..2]
}
return rets
}
def findMaxSimilar(ids, o1, datas){
def maxObj = null
def maxScore = -1000
for (def o2 : datas) {
if (o1._id.toString() != o2._id.toString() && o1.region==o2.region && !ids.contains(o2._id.toString())) {
def name1 = o1.name//.replace("小区","").replace("别墅","")
def name2 = o2.name//.replace("小区","").replace("别墅","")
def nameScore = CosineSimilarAlgorithm.levenshteinSimilarity(name1, name2);
def addrScore = null
if(o1.address && o2.address){
addrScore = CosineSimilarAlgorithm.levenshteinSimilarity(o1.address, o2.address);
}
def str = null
if(o1.name.length() > o2.name.length()){
str = o1.name.replace(o2.name, "")
}else{
str = o2.name.replace(o1.name, "")
}
if (nameScore > 0.6 && nameScore>maxScore && ["一期","二期","三期","四期","五期","六期","七期","八期","九期","十期","东区","西区","南区","北区","A区","B区","C区","D区"].contains(str)) {
maxObj = o2
o2.nameScore = nameScore
o2.addrScore = addrScore
o2.score = nameScore
}
}
}
return maxObj
}
def findOneSimilarOffice(o1, offices) {
def maxScore = 0
def maxObj = null
for(def o2 : offices){
if(o1._id.toString()!=o2._id.toString()){
for (def o2 : offices) {
if (o1._id.toString() != o2._id.toString()) {
double nameScore = CosineSimilarAlgorithm.getSimilarity(o1.name, o2.name);
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address,o2.address);
if(nameScore+addrScore>maxScore){
maxScore = nameScore+addrScore
double addrScore = CosineSimilarAlgorithm.getSimilarity(o1.address, o2.address);
if (nameScore + addrScore > maxScore) {
maxScore = nameScore + addrScore
maxObj = o2
}
}
}
def distance = -1000
if(o1.lat && maxObj && maxObj.lat){
distance = GeoHelper.GetDistance(o1.lat/10000000000.0, o1.lng/10000000000.0, maxObj.lat/10000000000.0, maxObj.lng/10000000000.0)
if (o1.lat && maxObj && maxObj.lat) {
distance = GeoHelper.GetDistance(o1.lat / 10000000000.0, o1.lng / 10000000000.0, maxObj.lat / 10000000000.0, maxObj.lng / 10000000000.0)
}
if(maxObj){
if (maxObj) {
maxObj.distance = distance
maxObj.score = maxScore
return maxObj
}
return null
}
def findBaiduDatas(city, district, name){
def q = new BasicDBObject()
q.put("city", city)
q.put("district", district)
q.put("name", name)
// q.put('std_tag', '房地产;住宅区')
def obj = BAIDU_POI_COLL.findOne(q)
return obj;
}
def addOffice(){
ExcelUtils.modify("D:\\数据\\高德待入库写字楼.xlsx",{d->
if(d['L']!='人工删除'){
def city = RegionData.formatCity(d['C'])
def region = RegionData.formatRegion(city, d['D'])
if(city && region){
def locationstr = d['G'].toString()+","+d['H'].toString()
def bdpoint = FangjiaClientUtils.convert(locationstr, 1)
def udata = new BasicDBObject()
if(bdpoint){
udata.put("lng2", (bdpoint[0]*10000000000).longValue())
udata.put("lat2", (bdpoint[1]*10000000000).longValue())
}
udata.put("city", city)
udata.put("region", region)
udata.put("name", d['E'])
udata.put("address", d['I'])
udata.put("cat", "office")
udata.put("status", 0)
udata.put("c_date",new Date())
udata.put("m_date",new Date())
udata.put("lng", (d['G'].toString().toDouble()*10000000000).longValue())
udata.put("lat", (d['H'].toString().toDouble()*10000000000).longValue())
def estate_type = null
if(d['M'] && d['M'].contains("园区")){
estate_type = "创意园区"
}
if(!estate_type && d['E'].contains("创意")){
estate_type = "创意园区"
}
if(estate_type){
udata.put("estate_type2", estate_type)
}
def markMap = [:]
markMap.put("data_source", "机器入库")
udata.put("mark_map", markMap)
DbUtils.seaweed.save(udata)
println d['E'] +" "+udata._id.toString()
}
}
})
}
def findNewFjwAvgPrice(def commid, def cityid){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujia_bds_id='${commid}' order by gujiabds_clc_yrmo_val desc limit 1"
def map = [:]
pgsqlInstance235.eachRow(sql, { r ->
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
})
return map
}
def findNewFjwAvgPriceList(def cityid, date){
def sql = "SELECT gujia_bds_clc_avprc,gujia_bds_clmtd,gujiabds_clc_yrmo_val,gujia_bds_cfdc_dgr,gujia_bds_id from price_stat.avm_community_base_price where urbn_id='${cityid}' and gujiabds_clc_yrmo_val=${date}"
def retMap = [:]
pgsqlInstance235.eachRow(sql, { r ->
def map = [:]
map.avg_price = r.gujia_bds_clc_avprc
map.type = r.gujia_bds_clmtd
map.month = r.gujiabds_clc_yrmo_val
map.credit_value = r.gujia_bds_cfdc_dgr
if(r.gujia_bds_clmtd == 1){
map.calc_type = "人工估值"
}else if(r.gujia_bds_clmtd == 2){
map.calc_type = "样本计算"
}else if(r.gujia_bds_clmtd == 3){
map.calc_type = "神经网络"
}
retMap.put(r.gujia_bds_id, map)
})
return retMap
}
}