+2
-2
@@ -80,9 +80,9 @@ class BigdataDealBatch extends BaseBatch{
|
|||||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||||
}
|
}
|
||||||
|
|
||||||
def fileName = "/home/goojia/ccb/bigdata_deal.dat"
|
def fileName = "/home/goojia/ccb/BIGDATA_DEAL_GJ.dat"
|
||||||
if(CommonDatautils.isWindow()){
|
if(CommonDatautils.isWindow()){
|
||||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat"
|
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_DEAL_GJ.dat"
|
||||||
}
|
}
|
||||||
|
|
||||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||||
|
|||||||
+34
-38
@@ -61,7 +61,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
private static def limit = 5000;
|
private static def limit = 5000;
|
||||||
|
|
||||||
def static final SHORT_DATE = Utils.getLastMonth()
|
def static final SHORT_DATE = Utils.getLastMonth()
|
||||||
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6]
|
def static final statusMap = ["城市未匹配":1,"区域未匹配":2,"小区匹配失败":3,"小区匹配成功":4,"异常错误":5, "无价格":6, "无名称":7]
|
||||||
|
|
||||||
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
def static final websites = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集", "链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"]
|
||||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||||
@@ -83,14 +83,13 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
|
|
||||||
@Override
|
@Override
|
||||||
void process(Object args) {
|
void process(Object args) {
|
||||||
|
|
||||||
def flag = true;
|
def flag = true;
|
||||||
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
// ThirdPartPriceDataRepeatBatchUtils.findThirdPriceDataRepeatBatch(SHORT_DATE, thirdPartyColl)
|
||||||
|
|
||||||
cityMap.each{city,alias->
|
cityMap.each{city,alias->
|
||||||
println "开始加载【${city}】"
|
println "开始加载【${city}】"
|
||||||
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
def detailMap = CommonDatautils.findAllCityDistrictNameAndAlias(city, "district", ["city","region","name","address","alias","addr_alias","mapping_alias"])
|
||||||
districtMap.putAll(detailMap)
|
districtMap.putAll(detailMap)
|
||||||
}
|
}
|
||||||
|
|
||||||
while(flag){
|
while(flag){
|
||||||
@@ -122,7 +121,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
void run() {
|
void run() {
|
||||||
try{
|
try{
|
||||||
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
println "(${rowIndex}-${count}-${count_.incrementAndGet()})【${dbo.city}-${dbo.region}-${dbo.name}-${dbo.address}-${dbo.source}】"
|
||||||
// if(dbo.avg_price){
|
if(dbo.name){
|
||||||
|
|
||||||
//标准化城市(包含region)
|
//标准化城市(包含region)
|
||||||
def data = standardCity(dbo);
|
def data = standardCity(dbo);
|
||||||
@@ -147,10 +146,10 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
// }else{
|
}else{
|
||||||
// saveLog(dbo, "无价格");
|
saveLog(dbo, "无名称");
|
||||||
// updateStatus(dbo,statusMap.get("无价格"))
|
updateStatus(dbo,statusMap.get("无名称"))
|
||||||
// }
|
}
|
||||||
|
|
||||||
count++;
|
count++;
|
||||||
}catch (Exception e){
|
}catch (Exception e){
|
||||||
@@ -480,27 +479,7 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
type = 12;//区域不同
|
type = 12;//区域不同
|
||||||
district = null;
|
district = null;
|
||||||
}else{
|
}else{
|
||||||
def score1 = 0;
|
type=15
|
||||||
if(data.address && district.address){
|
|
||||||
score1 = CosineSimilarAlgorithm.getSimilarityRatio(data.address, district.address);
|
|
||||||
}
|
|
||||||
if(score1 && score1 >=0.8){
|
|
||||||
type = 13;//地址相似度在0.8以上
|
|
||||||
}else{
|
|
||||||
if(addr && (district.address && addr.indexOf(district.address)>-1||district.address.toString().indexOf(addr)>-1) && (data.name.indexOf(district.name)>-1 || district.name.toString().indexOf(data.name)>-1)){
|
|
||||||
type = 13;
|
|
||||||
}else{
|
|
||||||
if(!addr){
|
|
||||||
def openDistName = district.name.replace("小区","")
|
|
||||||
def name = data.name.replace("小区","")
|
|
||||||
if(openDistName.contains(name) || name.contains(openDistName)){
|
|
||||||
type = 15;//匹配接口通过名称配上,名称包含
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
//type = 11;
|
|
||||||
district = null;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -607,11 +586,13 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
def getMatchNameFromMap(def city, def region, def disName, String address){
|
def getMatchNameFromMap(def city, def region, def disName, String address){
|
||||||
//查询房价网是否存在该小区
|
//查询房价网是否存在该小区
|
||||||
def dbo = findSameNameFromMap(city, region, disName);
|
def dbo = findSameNameFromMap(city, region, disName);
|
||||||
def addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
def addrFlag = null
|
||||||
if(addrFlag){
|
if(address){
|
||||||
dbo = addressCheck(dbo, region, address);
|
addrFlag = address.matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$')
|
||||||
|
if(addrFlag){
|
||||||
|
dbo = addressCheck(dbo, region, address);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if(!dbo){
|
if(!dbo){
|
||||||
//别名
|
//别名
|
||||||
def distrs = findAliasNameFromMap(city, region, disName);
|
def distrs = findAliasNameFromMap(city, region, disName);
|
||||||
@@ -683,10 +664,16 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
return dbo;
|
return dbo;
|
||||||
}*/
|
}*/
|
||||||
def getMatchAddressFromMap(def city, def region, def address, def name){
|
def getMatchAddressFromMap(def city, def region, def address, def name){
|
||||||
def dbo = findSameNameFromMap(city, region, address);
|
def dbo = null
|
||||||
|
if(address && isStandardRoad(address)){
|
||||||
|
dbo = findSameNameFromMap(city, region, address);
|
||||||
|
}
|
||||||
if(!dbo){
|
if(!dbo){
|
||||||
def result = findAddressFromMap(city, region, address);
|
def result = null
|
||||||
if(result.size()==1){
|
if(address && isStandardRoad(address)){
|
||||||
|
result = findAddressFromMap(city, region, address);
|
||||||
|
}
|
||||||
|
if(result && result.size()==1){
|
||||||
dbo = result[0];
|
dbo = result[0];
|
||||||
}else if(result && result.size()>1 && name){
|
}else if(result && result.size()>1 && name){
|
||||||
def score = null;
|
def score = null;
|
||||||
@@ -734,6 +721,15 @@ class DataStandardBatch extends BaseBatch{
|
|||||||
return dbo;
|
return dbo;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
def isStandardRoad(def s){
|
||||||
|
def pattern = Pattern.compile("[路|街|大道|巷]\\d+(号|弄)");
|
||||||
|
def matcher = pattern.matcher(s)
|
||||||
|
if(matcher.find()){
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
def addressCheck(def dbo, def region, def address){
|
def addressCheck(def dbo, def region, def address){
|
||||||
if(dbo && !region && address && address.length()>3){
|
if(dbo && !region && address && address.length()>3){
|
||||||
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){
|
if(dbo.address.contains(address)||address.contains(dbo.address)||dbo.address==address){
|
||||||
|
|||||||
Reference in New Issue
Block a user