重复小区去重

This commit is contained in:
2024-11-11 18:57:09 +08:00
parent 8390303ce8
commit b44435a0b9
@@ -0,0 +1,68 @@
package com.fangjia.script.gj.data_handle
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.CosineSimilarAlgorithm
import com.fangjia.script.util.ExcelUtils
/**
* 查找重复小区
*/
class DistrictRepeatBatch {
static main(args) {
def task = new DistrictRepeatBatch()
task.run()
}
def run(){
def rets = []
rets << ["id","城市","区域", "名称", "物业类型", "地址", "备注", "id(重复)","城市(重复)","区域(重复)", "名称(重复)", "物业类型(重复)", "地址(重复)", "相似度"]
def citys = CommonDatautils.findAllCity()
def index = 0
for(def city : citys){
index++
def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address","estate_type2","lat", "lng"])
def rows = findRepeatByEstateType(districts)
rets.addAll(rows)
println "【${index}】 "+city +" 完成比对"
}
ExcelUtils.write("D:\\数据\\小区重复数据处理.xlsx", rets, "小区")
}
def findRepeatByEstateType(districts){
def rows = []
def ids = []
for(def d1 : districts){
if(ids.contains(d1._id.toString())){
continue
}
for(def d2 : districts){
if(d1._id.toString() == d2._id.toString()){
continue
}
if(d1.estate_type2!=d2.estate_type2){
continue
}
def d1_name = d1.name.replace("别墅", "").replace("商住楼", "").replace("商住", "").replace("小区", "")
def d2_name = d2.name.replace("别墅", "").replace("商住楼", "").replace("商住", "").replace("小区", "")
if(d1_name != d2_name){
continue
}
def key1 = d1.region+"-"+d1_name+"-"+d1.address
def key2 = d2.region+"-"+d2_name+"-"+d2.address
def score = CosineSimilarAlgorithm.getSimilarityRatio(key1, key2);
if(score>0.9){
ids << d1._id.toString()
ids << d2._id.toString()
rows << [d1._id.toString(), d1.city, d1.region, d1.name, d1.estate_type2, d1.address, "物业类型", d2._id.toString(), d2.city, d2.region, d2.name, d2.estate_type2, d2.address, score]
println d1.region +" "+d1.name +" "+d1.address +" 重复数据("+d2.region+" "+d2.name+" "+d2.address+") score:"+score
}
}
}
return rows;
}
}