+2
-2
@@ -80,9 +80,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_deal.dat"
|
||||
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
|
||||
+34
-38
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
private static def limit = 5000;
|
||||
|
||||
def static final SHORT_DATE = Utils.getLastMonth()
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||
|
||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
@@ -83,14 +83,13 @@ class DataStandardBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
|
||||
def flag = true;
|
||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||
|
||||
cityMap.each{city,alias->
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
||||
districtMap.putAll(detailMap)
|
||||
println "开始加载【${city}】"
|
||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
||||
districtMap.putAll(detailMap)
|
||||
}
|
||||
|
||||
while(flag){
|
||||
@@ -122,7 +121,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||
// if(dbo.avg_price){
|
||||
if(dbo.name){
|
||||
|
||||
//标准化城市(包含region)
|
||||
def data = standardCity(dbo);
|
||||
@@ -147,10 +146,10 @@ class DataStandardBatch extends BaseBatch{
|
||||
}
|
||||
|
||||
}
|
||||
// }else{
|
||||
// saveLog(dbo, "无价格");
|
||||
// updateStatus(dbo,statusMap.get("无价格"))
|
||||
// }
|
||||
}else{
|
||||
saveLog(dbo, "无名称");
|
||||
updateStatus(dbo,statusMap.get("无名称"))
|
||||
}
|
||||
|
||||
count++;
|
||||
}catch (Exception e){
|
||||
@@ -480,27 +479,7 @@ class DataStandardBatch extends BaseBatch{
|
||||
type = 12;//区域不同
|
||||
district = null;
|
||||
}else{
|
||||
def score1 = 0;
|
||||
if(data.address && district.address){
|
||||
score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address);
|
||||
}
|
||||
if(score1 && score1 >=0.8){
|
||||
type = 13;//地址相似度在0.8以上
|
||||
}else{
|
||||
if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){
|
||||
type = 13;
|
||||
}else{
|
||||
if(!addr){
|
||||
def openDistName = district.name.replace("小区","")
|
||||
def name = data.name.replace("小区","")
|
||||
if(openDistName.contains(name) || name.contains(openDistName)){
|
||||
type = 15;//匹配接口通过名称配上,名称包含
|
||||
}
|
||||
}
|
||||
}
|
||||
//type = 11;
|
||||
district = null;
|
||||
}
|
||||
type=15
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -607,11 +586,13 @@ class DataStandardBatch extends BaseBatch{
|
||||
def getMatchNameFromMap(def city, def region, def disName, String address){
|
||||
//查询房价网是否存在该小区
|
||||
def dbo = findSameNameFromMap(city, region, disName);
|
||||
def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(addrFlag){
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
def addrFlag = null
|
||||
if(address){
|
||||
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||
if(addrFlag){
|
||||
dbo = addressCheck(dbo, region, address);
|
||||
}
|
||||
}
|
||||
|
||||
if(!dbo){
|
||||
//别名
|
||||
def distrs = findAliasNameFromMap(city, region, disName);
|
||||
@@ -683,10 +664,16 @@ class DataStandardBatch extends BaseBatch{
|
||||
return dbo;
|
||||
}*/
|
||||
def getMatchAddressFromMap(def city, def region, def address, def name){
|
||||
def dbo = findSameNameFromMap(city, region, address);
|
||||
def dbo = null
|
||||
if(address && isStandardRoad(address)){
|
||||
dbo = findSameNameFromMap(city, region, address);
|
||||
}
|
||||
if(!dbo){
|
||||
def result = findAddressFromMap(city, region, address);
|
||||
if(result.size()==1){
|
||||
def result = null
|
||||
if(address && isStandardRoad(address)){
|
||||
result = findAddressFromMap(city, region, address);
|
||||
}
|
||||
if(result && result.size()==1){
|
||||
dbo = result[0];
|
||||
}else if(result && result.size()>1 && name){
|
||||
def score = null;
|
||||
@@ -734,6 +721,15 @@ class DataStandardBatch extends BaseBatch{
|
||||
return dbo;
|
||||
}
|
||||
|
||||
def isStandardRoad(def s){
|
||||
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||
def matcher = pattern.matcher(s)
|
||||
if(matcher.find()){
|
||||
return true;
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
def addressCheck(def dbo, def region, def address){
|
||||
if(dbo && !region && address && address.length()>3){
|
||||
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){
|
||||
|
||||
Reference in New Issue
Block a user