Merge branch 'script_gj' into 'master'

update

See merge request !128
This commit is contained in:
2021-09-24 17:43:19 +08:00
2 changed files with 36 additions and 40 deletions
@@ -80,9 +80,9 @@ class BigdataDealBatch extends BaseBatch{
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
def fileName = "/home/goojia/ccb/bigdata_deal.dat"
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
private static def limit = 5000;
def static final SHORT_DATE = Utils.getLastMonth()
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
@@ -83,14 +83,13 @@ class DataStandardBatch extends BaseBatch{
@Override
void process(Object args) {
def flag = true;
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
cityMap.each{city,alias->
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
districtMap.putAll(detailMap)
println "开始加载【${city}】"
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
districtMap.putAll(detailMap)
}
while(flag){
@@ -122,7 +121,7 @@ class DataStandardBatch extends BaseBatch{
void run() {
try{
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
// if(dbo.avg_price){
if(dbo.name){
//标准化城市(包含region)
def data = standardCity(dbo);
@@ -147,10 +146,10 @@ class DataStandardBatch extends BaseBatch{
}
}
// }else{
// saveLog(dbo, "无价格");
// updateStatus(dbo,statusMap.get("无价格"))
// }
}else{
saveLog(dbo, "无名称");
updateStatus(dbo,statusMap.get("无名称"))
}
count++;
}catch (Exception e){
@@ -480,27 +479,7 @@ class DataStandardBatch extends BaseBatch{
type = 12;//区域不同
district = null;
}else{
def score1 = 0;
if(data.address && district.address){
score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address);
}
if(score1 && score1 >=0.8){
type = 13;//地址相似度在0.8以上
}else{
if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){
type = 13;
}else{
if(!addr){
def openDistName = district.name.replace("小区","")
def name = data.name.replace("小区","")
if(openDistName.contains(name) || name.contains(openDistName)){
type = 15;//匹配接口通过名称配上,名称包含
}
}
}
//type = 11;
district = null;
}
type=15
}
}
}
@@ -607,11 +586,13 @@ class DataStandardBatch extends BaseBatch{
def getMatchNameFromMap(def city, def region, def disName, String address){
//查询房价网是否存在该小区
def dbo = findSameNameFromMap(city, region, disName);
def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
def addrFlag = null
if(address){
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
if(addrFlag){
dbo = addressCheck(dbo, region, address);
}
}
if(!dbo){
//别名
def distrs = findAliasNameFromMap(city, region, disName);
@@ -683,10 +664,16 @@ class DataStandardBatch extends BaseBatch{
return dbo;
}*/
def getMatchAddressFromMap(def city, def region, def address, def name){
def dbo = findSameNameFromMap(city, region, address);
def dbo = null
if(address && isStandardRoad(address)){
dbo = findSameNameFromMap(city, region, address);
}
if(!dbo){
def result = findAddressFromMap(city, region, address);
if(result.size()==1){
def result = null
if(address && isStandardRoad(address)){
result = findAddressFromMap(city, region, address);
}
if(result && result.size()==1){
dbo = result[0];
}else if(result && result.size()>1 && name){
def score = null;
@@ -734,6 +721,15 @@ class DataStandardBatch extends BaseBatch{
return dbo;
}
def isStandardRoad(def s){
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
def matcher = pattern.matcher(s)
if(matcher.find()){
return true;
}
return false
}
def addressCheck(def dbo, def region, def address){
if(dbo && !region && address && address.length()>3){
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){