Merge branch 'script_gj' into 'master'

update

See merge request !128
This commit is contained in:
2021-09-24 17:43:19 +08:00
2 changed files with 36 additions and 40 deletions
@@ -80,9 +80,9 @@ class BigdataDealBatch extends BaseBatch{
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
} }
def fileName = "/home/goojia/ccb/bigdata_deal.dat" def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){ if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat" fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
} }
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
private static def limit = 5000; private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth() def static final SHORT_DATE = Utils.getLastMonth()
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6] def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -83,14 +83,13 @@ class DataStandardBatch extends BaseBatch{
@Override @Override
void process(Object args) { void process(Object args) {
def flag = true; def flag = true;
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias-> cityMap.each{city,alias->
println "开始加载【${city}】" println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
districtMap.putAll(detailMap) districtMap.putAll(detailMap)
} }
while(flag){ while(flag){
@@ -122,7 +121,7 @@ class DataStandardBatch extends BaseBatch{
void run() { void run() {
try{ try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
// if(dbo.avg_price){ if(dbo.name){
//标准化城市(包含region) //标准化城市(包含region)
def data = standardCity(dbo); def data = standardCity(dbo);
@@ -147,10 +146,10 @@ class DataStandardBatch extends BaseBatch{
} }
} }
// }else{ }else{
// saveLog(dbo, "无价格"); saveLog(dbo, "无名称");
// updateStatus(dbo,statusMap.get("无价格")) updateStatus(dbo,statusMap.get("无名称"))
// } }
count++; count++;
}catch (Exception e){ }catch (Exception e){
@@ -480,27 +479,7 @@ class DataStandardBatch extends BaseBatch{
type = 12;//区域不同 type = 12;//区域不同
district = null; district = null;
}else{ }else{
def score1 = 0; type=15
if(data.address && district.address){
score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address);
}
if(score1 && score1 >=0.8){
type = 13;//地址相似度在0.8以上
}else{
if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){
type = 13;
}else{
if(!addr){
def openDistName = district.name.replace("小区","")
def name = data.name.replace("小区","")
if(openDistName.contains(name) || name.contains(openDistName)){
type = 15;//匹配接口通过名称配上,名称包含
}
}
}
//type = 11;
district = null;
}
} }
} }
} }
@@ -607,11 +586,13 @@ class DataStandardBatch extends BaseBatch{
def getMatchNameFromMap(def city, def region, def disName, String address){ def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区 //查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName); def dbo = findSameNameFromMap(city, region, disName);
def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') def addrFlag = null
if(addrFlag){ if(address){
dbo = addressCheck(dbo, region, address); addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
}
} }
if(!dbo){ if(!dbo){
//别名 //别名
def distrs = findAliasNameFromMap(city, region, disName); def distrs = findAliasNameFromMap(city, region, disName);
@@ -683,10 +664,16 @@ class DataStandardBatch extends BaseBatch{
return dbo; return dbo;
}*/ }*/
def getMatchAddressFromMap(def city, def region, def address, def name){ def getMatchAddressFromMap(def city, def region, def address, def name){
def dbo = findSameNameFromMap(city, region, address); def dbo = null
if(address && isStandardRoad(address)){
dbo = findSameNameFromMap(city, region, address);
}
if(!dbo){ if(!dbo){
def result = findAddressFromMap(city, region, address); def result = null
if(result.size()==1){ if(address && isStandardRoad(address)){
result = findAddressFromMap(city, region, address);
}
if(result && result.size()==1){
dbo = result[0]; dbo = result[0];
}else if(result && result.size()>1 && name){ }else if(result && result.size()>1 && name){
def score = null; def score = null;
@@ -734,6 +721,15 @@ class DataStandardBatch extends BaseBatch{
return dbo; return dbo;
} }
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
def addressCheck(def dbo, def region, def address){ def addressCheck(def dbo, def region, def address){
if(dbo && !region && address && address.length()>3){ if(dbo && !region && address && address.length()>3){
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){ if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){