This commit is contained in:
2021-09-10 11:00:39 +08:00
parent 924f9754b9
commit 5a1844b926
@@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.district.NameProcess
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
import com.fangjia.script.linxsh.util.CommonUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.CosineSimilarAlgorithm
@@ -86,9 +88,9 @@ class DataStandardBatch extends BaseBatch{
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias->
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
districtMap.putAll(detailMap)
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
districtMap.putAll(detailMap)
}
while(flag){
@@ -126,15 +128,24 @@ class DataStandardBatch extends BaseBatch{
def data = standardCity(dbo);
if(data){
//匹配区域
data = standardRegion(data);
if(data){
//标准化小区名
def data3 = standardDistrict(data, dbo.avg_price);
if(data.city=="三亚" && data.region=="陵水"){
data.city = "陵水"
data.region = ""
def data3 = standardDistrictSimple(data, dbo.avg_price);
if(data3){
successCount++;
}
}else if(data){
data = standardRegion(data);
if(data){
//标准化小区名
def data3 = standardDistrict(data, dbo.avg_price);
if(data3){
successCount++;
}
}
}
}
// }else{
// saveLog(dbo, "无价格");
@@ -253,6 +264,104 @@ class DataStandardBatch extends BaseBatch{
return data;
}
def standardDistrict2(def data, def avg_price){
def newName = NameProcess.nameStandard(data.city, data.region, data.name)
def district = null
if(newName){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address);
if(district){
}
}
if(district && avg_price){
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
saveSuccessLog(data, district, type);
updateStatus(data,statusMap.get("小区匹配成功"))
println district.city+" "+district.name+" success"
}else{
if(district){
// data.type = type
saveLog(data, "无价格");
updateStatus(data,statusMap.get("无价格"))
println data.city+" "+data.name+" fail(无价格)"
}else{
println "[匹配失败]"
saveLog(data, "小区匹配失败");
updateStatus(data,statusMap.get("小区匹配失败"))
println data.city+" "+data.name+" fail"
}
}
return district;
}
def standardDistrictSimple(def data, def avg_price){
def newName1 = null
def newName2 = null
if(data.name.contains("别名")){
newName1 = data.name.toString().substring(0, data.name.toString().indexOf("别名")-1)
newName2 = data.name.toString().substring(data.name.toString().indexOf("别名")+3)
}else{
newName1 = NameProcess.nameStandard(data.city, data.region, data.name)
}
def district = null
def type = null
if(newName1){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address);
if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){
district._id = data._id
type = 1
}else{
if(newName2){
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address);
if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){
district._id = data._id
type = 1
}else{
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2)
if(score>0.8){
district._id = data._id
type = 11
}else{
district = null
}
}
}else{
if(district){
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1)
if(score>0.8){
district._id = data._id
type = 11
}else{
district = null
}
}
}
}
}
if(district && avg_price){
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
saveSuccessLog(data, district, type);
updateStatus(data,statusMap.get("小区匹配成功"))
println district.city+" "+district.name+" success"
}else{
if(district){
data.type = type
saveLog(data, "无价格");
updateStatus(data,statusMap.get("无价格"))
println data.city+" "+data.name+" fail(无价格)"
}else{
println "[匹配失败]"
saveLog(data, "小区匹配失败");
updateStatus(data,statusMap.get("小区匹配失败"))
println data.city+" "+data.name+" fail"
}
}
return district;
}
def standardDistrict(def data, def avg_price){
//名称匹配,名称别名匹配,地址别名匹配
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
@@ -368,7 +477,7 @@ class DataStandardBatch extends BaseBatch{
}
if(district){
if(data.region != district.region){
//type = 12;//区域不同
type = 12;//区域不同
district = null;
}else{
def score1 = 0;
@@ -495,10 +604,14 @@ class DataStandardBatch extends BaseBatch{
}*/
def getMatchNameFromMap(def city, def region, def disName, def address){
def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName);
dbo = addressCheck(dbo, region, address);
def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
}
if(!dbo){
//别名
def distrs = findAliasNameFromMap(city, region, disName);
@@ -506,7 +619,7 @@ class DataStandardBatch extends BaseBatch{
dbo = distrs[0]
}else if(distrs && distrs.size()>1 && address){
for(def d : distrs){
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
dbo = d;
break;
}