update
This commit is contained in:
+125
-12
@@ -4,6 +4,8 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.district.NameProcess
|
||||
import com.fangjia.script.gj.district.SeaweedDataMatchProcess
|
||||
import com.fangjia.script.linxsh.util.CommonUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.CosineSimilarAlgorithm
|
||||
@@ -86,9 +88,9 @@ class DataStandardBatch extends BaseBatch{
|
||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
||||
districtMap.putAll(detailMap)
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -126,15 +128,24 @@ class DataStandardBatch extends BaseBatch{
|
||||
def data = standardCity(dbo);
|
||||
if(data){
|
||||
//匹配区域
|
||||
data = standardRegion(data);
|
||||
|
||||
if(data){
|
||||
//标准化小区名
|
||||
def data3 = standardDistrict(data, dbo.avg_price);
|
||||
if(data.city=="三亚" && data.region=="陵水"){
|
||||
data.city = "陵水"
|
||||
data.region = ""
|
||||
def data3 = standardDistrictSimple(data, dbo.avg_price);
|
||||
if(data3){
|
||||
successCount++;
|
||||
}
|
||||
}else if(data){
|
||||
data = standardRegion(data);
|
||||
if(data){
|
||||
//标准化小区名
|
||||
def data3 = standardDistrict(data, dbo.avg_price);
|
||||
if(data3){
|
||||
successCount++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
// }else{
|
||||
// saveLog(dbo, "无价格");
|
||||
@@ -253,6 +264,104 @@ class DataStandardBatch extends BaseBatch{
|
||||
return data;
|
||||
}
|
||||
|
||||
def standardDistrict2(def data, def avg_price){
|
||||
def newName = NameProcess.nameStandard(data.city, data.region, data.name)
|
||||
def district = null
|
||||
if(newName){
|
||||
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, data.name, data.address);
|
||||
if(district){
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
if(district && avg_price){
|
||||
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
|
||||
saveSuccessLog(data, district, type);
|
||||
updateStatus(data,statusMap.get("小区匹配成功"))
|
||||
println district.city+" "+district.name+" success"
|
||||
}else{
|
||||
if(district){
|
||||
// data.type = type
|
||||
saveLog(data, "无价格");
|
||||
updateStatus(data,statusMap.get("无价格"))
|
||||
println data.city+" "+data.name+" fail(无价格)"
|
||||
}else{
|
||||
println "[匹配失败]"
|
||||
saveLog(data, "小区匹配失败");
|
||||
updateStatus(data,statusMap.get("小区匹配失败"))
|
||||
println data.city+" "+data.name+" fail"
|
||||
}
|
||||
}
|
||||
return district;
|
||||
|
||||
}
|
||||
|
||||
def standardDistrictSimple(def data, def avg_price){
|
||||
def newName1 = null
|
||||
def newName2 = null
|
||||
if(data.name.contains("别名")){
|
||||
newName1 = data.name.toString().substring(0, data.name.toString().indexOf("别名")-1)
|
||||
newName2 = data.name.toString().substring(data.name.toString().indexOf("别名")+3)
|
||||
}else{
|
||||
newName1 = NameProcess.nameStandard(data.city, data.region, data.name)
|
||||
}
|
||||
def district = null
|
||||
def type = null
|
||||
if(newName1){
|
||||
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName1, data.address);
|
||||
if(district && (district.name == newName1 || (district.alias && district.alias.contains(newName1)))){
|
||||
district._id = data._id
|
||||
type = 1
|
||||
}else{
|
||||
if(newName2){
|
||||
district = SeaweedDataMatchProcess.doMatch(data.city, data.region, newName2, data.address);
|
||||
if(district && (district.name == newName2 || (district.alias && district.alias.contains(newName2)))){
|
||||
district._id = data._id
|
||||
type = 1
|
||||
}else{
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName2)
|
||||
if(score>0.8){
|
||||
district._id = data._id
|
||||
type = 11
|
||||
}else{
|
||||
district = null
|
||||
}
|
||||
}
|
||||
}else{
|
||||
if(district){
|
||||
def score = CosineSimilarAlgorithm.getSimilarity(district.name, newName1)
|
||||
if(score>0.8){
|
||||
district._id = data._id
|
||||
type = 11
|
||||
}else{
|
||||
district = null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(district && avg_price){
|
||||
removeErrorLog(data);//删除错误日志表中区域不匹配的,正式运行时删除
|
||||
saveSuccessLog(data, district, type);
|
||||
updateStatus(data,statusMap.get("小区匹配成功"))
|
||||
println district.city+" "+district.name+" success"
|
||||
}else{
|
||||
if(district){
|
||||
data.type = type
|
||||
saveLog(data, "无价格");
|
||||
updateStatus(data,statusMap.get("无价格"))
|
||||
println data.city+" "+data.name+" fail(无价格)"
|
||||
}else{
|
||||
println "[匹配失败]"
|
||||
saveLog(data, "小区匹配失败");
|
||||
updateStatus(data,statusMap.get("小区匹配失败"))
|
||||
println data.city+" "+data.name+" fail"
|
||||
}
|
||||
}
|
||||
return district;
|
||||
}
|
||||
|
||||
def standardDistrict(def data, def avg_price){
|
||||
//名称匹配,名称别名匹配,地址别名匹配
|
||||
//def district = getMatchName(data.city, data.region, data.name, data.address,"district")
|
||||
@@ -368,7 +477,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
if(district){
|
||||
if(data.region != district.region){
|
||||
//type = 12;//区域不同
|
||||
type = 12;//区域不同
|
||||
district = null;
|
||||
}else{
|
||||
def score1 = 0;
|
||||
@@ -495,10 +604,14 @@ class DataStandardBatch extends BaseBatch{
|
||||
}*/
|
||||
|
||||
|
||||
def getMatchNameFromMap(def city, def region, def disName, def address){
|
||||
def getMatchNameFromMap(def city, def region, def disName, String address){
|
||||
//查询房价网是否存在该小区
|
||||
def dbo = findSameNameFromMap(city, region, disName);
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(addrFlag){
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
}
|
||||
|
||||
if(!dbo){
|
||||
//别名
|
||||
def distrs = findAliasNameFromMap(city, region, disName);
|
||||
@@ -506,7 +619,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
dbo = distrs[0]
|
||||
}else if(distrs && distrs.size()>1 && address){
|
||||
for(def d : distrs){
|
||||
if(d.address.contains(address)||d.address==address || address.contains(d.address)){
|
||||
if(addrFlag && (d.address.contains(address)||d.address==address || address.contains(d.address))){
|
||||
dbo = d;
|
||||
break;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user