diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 0fb1a6a97..ec1b947cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -80,9 +80,9 @@ class BigdataDealBatch extends BaseBatch{ // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") } - def fileName = "/home/goojia/ccb/bigdata_deal.dat" + def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat" } FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index cbe499054..5522782e9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{ private static def limit = 5000; def static final SHORT_DATE = Utils.getLastMonth() - def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6] + def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7] def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) @@ -83,14 +83,13 @@ class DataStandardBatch extends BaseBatch{ @Override void process(Object args) { - def flag = true; // ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl) cityMap.each{city,alias-> - println "开始加载【${city}】" - def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) - districtMap.putAll(detailMap) + println "开始加载【${city}】" + def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"]) + districtMap.putAll(detailMap) } while(flag){ @@ -122,7 +121,7 @@ class DataStandardBatch extends BaseBatch{ void run() { try{ println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】" -// if(dbo.avg_price){ + if(dbo.name){ //标准化城市(包含region) def data = standardCity(dbo); @@ -147,10 +146,10 @@ class DataStandardBatch extends BaseBatch{ } } -// }else{ -// saveLog(dbo, "无价格"); -// updateStatus(dbo,statusMap.get("无价格")) -// } + }else{ + saveLog(dbo, "无名称"); + updateStatus(dbo,statusMap.get("无名称")) + } count++; }catch (Exception e){ @@ -480,27 +479,7 @@ class DataStandardBatch extends BaseBatch{ type = 12;//区域不同 district = null; }else{ - def score1 = 0; - if(data.address && district.address){ - score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address); - } - if(score1 && score1 >=0.8){ - type = 13;//地址相似度在0.8以上 - }else{ - if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){ - type = 13; - }else{ - if(!addr){ - def openDistName = district.name.replace("小区","") - def name = data.name.replace("小区","") - if(openDistName.contains(name) || name.contains(openDistName)){ - type = 15;//匹配接口通过名称配上,名称包含 - } - } - } - //type = 11; - district = null; - } + type=15 } } } @@ -607,11 +586,13 @@ class DataStandardBatch extends BaseBatch{ def getMatchNameFromMap(def city, def region, def disName, String address){ //查询房价网是否存在该小区 def dbo = findSameNameFromMap(city, region, disName); - def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') - if(addrFlag){ - dbo = addressCheck(dbo, region, address); + def addrFlag = null + if(address){ + addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(addrFlag){ + dbo = addressCheck(dbo, region, address); + } } - if(!dbo){ //别名 def distrs = findAliasNameFromMap(city, region, disName); @@ -683,10 +664,16 @@ class DataStandardBatch extends BaseBatch{ return dbo; }*/ def getMatchAddressFromMap(def city, def region, def address, def name){ - def dbo = findSameNameFromMap(city, region, address); + def dbo = null + if(address && isStandardRoad(address)){ + dbo = findSameNameFromMap(city, region, address); + } if(!dbo){ - def result = findAddressFromMap(city, region, address); - if(result.size()==1){ + def result = null + if(address && isStandardRoad(address)){ + result = findAddressFromMap(city, region, address); + } + if(result && result.size()==1){ dbo = result[0]; }else if(result && result.size()>1 && name){ def score = null; @@ -734,6 +721,15 @@ class DataStandardBatch extends BaseBatch{ return dbo; } + def isStandardRoad(def s){ + def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)"); + def matcher = pattern.matcher(s) + if(matcher.find()){ + return true; + } + return false + } + def addressCheck(def dbo, def region, def address){ if(dbo && !region && address && address.length()>3){ if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){