From 5a1844b9267520d9caeb87158bf0f8bb6019bff9 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 10 Sep 2021 11:00:39 +0800 Subject: [PATCH] update --- .../thirdPriceCrawl/DataStandardBatch.groovy | 137 ++++++++++++++++-- 1 file changed, 125 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index c8ab1b6d8..cbe499054 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.linxsh.util.CommonUtil import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CosineSimilarAlgorithm @@ -86,9 +88,9 @@ class DataStandardBatch extends BaseBatch{ // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) - districtMap.putAll(detailMap) + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + districtMap.putAll(detailMap) } while(flag){ @@ -126,15 +128,24 @@ class DataStandardBatch extends BaseBatch{ def data = standardCity(dbo); if(data){ //匹配区域 - data = standardRegion(data); - - if(data){ - //标准化小区名 - def data3 = standardDistrict(data, dbo.avg_price); + if(data.city=="三亚" && data.region=="陵水"){ + data.city = "陵水" + data.region = "" + def data3 = standardDistrictSimple(data, dbo.avg_price); if(data3){ successCount++; } + }else if(data){ + data = standardRegion(data); + if(data){ + //标准化小区名 + def data3 = standardDistrict(data, dbo.avg_price); + if(data3){ + successCount++; + } + } } + } // }else{ // saveLog(dbo, "无价格"); @@ -253,6 +264,104 @@ class DataStandardBatch extends BaseBatch{ return data; } + def standardDistrict2(def data, def avg_price){ + def newName = NameProcess.nameStandard(data.city, data.region, data.name) + def district = null + if(newName){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address); + if(district){ + + } + } + + if(district && avg_price){ + removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除 + saveSuccessLog(data, district, type); + updateStatus(data,statusMap.get("小区匹配成功")) + println district.city+" "+district.name+" success" + }else{ + if(district){ +// data.type = type + saveLog(data, "无价格"); + updateStatus(data,statusMap.get("无价格")) + println data.city+" "+data.name+" fail(无价格)" + }else{ + println "[匹配失败]" + saveLog(data, "小区匹配失败"); + updateStatus(data,statusMap.get("小区匹配失败")) + println data.city+" "+data.name+" fail" + } + } + return district; + + } + + def standardDistrictSimple(def data, def avg_price){ + def newName1 = null + def newName2 = null + if(data.name.contains("别名")){ + newName1 = data.name.toString().substring(0, data.name.toString().indexOf("别名")-1) + newName2 = data.name.toString().substring(data.name.toString().indexOf("别名")+3) + }else{ + newName1 = NameProcess.nameStandard(data.city, data.region, data.name) + } + def district = null + def type = null + if(newName1){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address); + if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){ + district._id = data._id + type = 1 + }else{ + if(newName2){ + district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address); + if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){ + district._id = data._id + type = 1 + }else{ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2) + if(score>0.8){ + district._id = data._id + type = 11 + }else{ + district = null + } + } + }else{ + if(district){ + def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1) + if(score>0.8){ + district._id = data._id + type = 11 + }else{ + district = null + } + } + } + } + } + + if(district && avg_price){ + removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除 + saveSuccessLog(data, district, type); + updateStatus(data,statusMap.get("小区匹配成功")) + println district.city+" "+district.name+" success" + }else{ + if(district){ + data.type = type + saveLog(data, "无价格"); + updateStatus(data,statusMap.get("无价格")) + println data.city+" "+data.name+" fail(无价格)" + }else{ + println "[匹配失败]" + saveLog(data, "小区匹配失败"); + updateStatus(data,statusMap.get("小区匹配失败")) + println data.city+" "+data.name+" fail" + } + } + return district; + } + def standardDistrict(def data, def avg_price){ //名称匹配,名称别名匹配,地址别名匹配 //def district = getMatchName(data.city, data.region, data.name, data.address,"district") @@ -368,7 +477,7 @@ class DataStandardBatch extends BaseBatch{ } if(district){ if(data.region != district.region){ - //type = 12;//区域不同 + type = 12;//区域不同 district = null; }else{ def score1 = 0; @@ -495,10 +604,14 @@ class DataStandardBatch extends BaseBatch{ }*/ - def getMatchNameFromMap(def city, def region, def disName, def address){ + def getMatchNameFromMap(def city, def region, def disName, String address){ //查询房价网是否存在该小区 def dbo = findSameNameFromMap(city, region, disName); - dbo = addressCheck(dbo, region, address); + def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(addrFlag){ + dbo = addressCheck(dbo, region, address); + } + if(!dbo){ //别名 def distrs = findAliasNameFromMap(city, region, disName); @@ -506,7 +619,7 @@ class DataStandardBatch extends BaseBatch{ dbo = distrs[0] }else if(distrs && distrs.size()>1 && address){ for(def d : distrs){ - if(d.address.contains(address)||d.address==address || address.contains(d.address)){ + if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){ dbo = d; break; }