diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index e3a690358..96bbe097f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1769,6 +1769,7 @@ public class AddressController { // matchParams.setUseMap(true); matchParams.setUseAliyunApi(true); +// matchParams.setUseCrf(true); matchParams.setContainsLocation(model.isContainsLocation()); Building building = addressParserService.standard(matchParams); Address address = building.getSearchAddress(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 8bc805491..9eb8f4114 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,14 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.CNNMFilter; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.GeoUtils; -import com.fangjia.core.util.StringUtil; +import com.fangjia.core.util.*; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; +import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -25,6 +23,7 @@ import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.bson.types.ObjectId; @@ -134,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService { formatAddress(building); } return building; + /*JSONObject result = getDataFromCRF("test"); + return new Building();*/ } @Override @@ -165,6 +166,21 @@ public class AddressParserServiceImpl implements AddressParserService { return building; } Building matchedBuilding; + //调用CRF算法切分数据 + /*JSONObject jsonObj = null; + if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + jsonObj = getDataFromCRF(addr.getOrgAddress()); + if(jsonObj != null){ + String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") + + (jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") + + (jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : ""); + if(StringUtils.isNotBlank(poi)){ + if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){ + addr.setKeywords(poi); + } + } + } + }*/ //keywords使用aliyun接口返回的poi AliyunBo aliyunBo = null; if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ @@ -176,6 +192,9 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setKeywords(poi); } } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(aliyunBo.getCity()); + } if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){ String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length())); if(StringUtils.isNotBlank(str)){ @@ -187,26 +206,80 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - if(StringUtils.isNotBlank(aliyunBo.getCity())){ - addr.setSuggestCity(aliyunBo.getCity()); - } - } - } - AddressMatchMetaData meta = null; - if(StringUtils.isBlank(addr.getCity())){ - addr.addKeyword(addr.getOrgAddress()); - meta = getMapMeta(addr); - if(meta != null){ - addr.setProvince(meta.getMapProvince()); - addr.setCity(meta.getMapCity()); - addr.setDistrict(meta.getDistrict()); } } + addr.addKeyword(addr.getOrgAddress()); if(params.isUseMap()){ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP); }else{ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB); } + + /*if(jsonObj != null){ + //乡镇优化 + String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : ""; + if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){ + String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township); + if(StringUtils.isNotBlank(town)){ + addr.setTownship(town); + } + } + //道路优化(两边均去掉乡镇) + String street = addr.getStreet(); + String streetNumber = addr.getStreetNumber(); + String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : ""; + String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : ""; + //去掉路名中含有乡镇名的 + if(StringUtils.isNotBlank(addr.getTownship())){ + street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street; + streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf; + } + + //crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理 + if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){ + //Java解析出路名为空 + if(StringUtils.isBlank(street)){ + street = streetCrf; + if(StringUtils.isNotBlank(streetNumberCrf)){ + streetNumber = streetNumberCrf; + } + }else{ + //CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则 + if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){ + street = streetCrf; + } + } + addr.setStreet(street); + addr.setStreetNumber(streetNumber); + } + //poi处理 + //没有匹配上库中小区时 + if(StringUtils.isBlank(matchedBuilding.getId())){ + String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") + + (jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") + + (jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : ""); + //去掉路名中含有乡镇名和路名 getParsedPropertyName + if(StringUtils.isNotBlank(addr.getTownship())){ + poi = poi.replace(addr.getTownship(),""); + } + if(StringUtils.isNotBlank(addr.getStreet())){ + poi = poi.replace(addr.getStreet(),""); + } + addr.setParsedPropertyName(poi); + } + //楼栋、单元、室号 + String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + addr.setOrgBuildingNumber(houseNo); + addr.setBuildingNumber(houseNo); + addr.setOrgUnitNumber(cellNo); + addr.setUnitNumber(cellNo); + addr.setFloor(floorNo); + addr.setOrgRoomNumber(roomNo); + addr.setRoomNumber(roomNo); + }*/ matchedBuilding.setProvince(addr.getProvince()); matchedBuilding.setSearchAddress(addr); matchedBuilding.setSuggestName(getSuggestName(matchedBuilding)); @@ -215,7 +288,8 @@ public class AddressParserServiceImpl implements AddressParserService { completBuildingField(matchedBuilding, aliyunBo, params); } Address tempAddr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(tempAddr.getStreet()) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + AddressMatchMetaData meta = getMapMeta(addr); if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){ matchedBuilding.setAddress(meta.getAddress()); } @@ -255,6 +329,25 @@ public class AddressParserServiceImpl implements AddressParserService { } return null; } + + /** + * TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用 + * @param address + * @return + */ + /*private JSONObject getDataFromCRF(String address){ + HttpRequest request = new HttpRequest(); + Map params = new HashMap<>(); + params.put("address",address); + try { + JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8"); + System.out.println(resp); + return resp; + }catch(Exception e){ + + } + return null; + }*/ /** * 信息不全时使用阿里接口补全数据 * @param matchedBuilding @@ -271,28 +364,39 @@ public class AddressParserServiceImpl implements AddressParserService { } } Address addr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(addr.getProvince())){ + if(StringUtils.isNotBlank(aliyunBo.getProv())){ + addr.setProvince(formatProvince(aliyunBo.getProv())); + } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } + if(StringUtils.isNotBlank(aliyunBo.getDistrict())){ + addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); + } + addr.setTownship(aliyunBo.getTown()); + + /*if(StringUtils.isBlank(addr.getProvince())){ addr.setProvince(formatProvince(aliyunBo.getProv())); } if(StringUtils.isBlank(addr.getCity())){ addr.setCity(RegionData.formatCity(aliyunBo.getCity())); } - - if(StringUtils.isBlank(addr.getDevelopmentZone())){ - addr.setDevelopmentZone(aliyunBo.getDevzone()); - } if(StringUtils.isBlank(addr.getDistrict())){ addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); } if(StringUtils.isBlank(addr.getTownship())){ addr.setTownship(aliyunBo.getTown()); + }*/ + if(StringUtils.isBlank(addr.getDevelopmentZone())){ + addr.setDevelopmentZone(aliyunBo.getDevzone()); } + /*addr.setOrgBuildingNumber(aliyunBo.getHouseno()); addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); addr.setUnitNumber(aliyunBo.getCellno());*/ if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ - addr.setRoomNumber(aliyunBo.getRoomno()); + addr.setOrgRoomNumber(aliyunBo.getRoomno()); addr.setRoomNumber(aliyunBo.getRoomno()); }else{ addr.setRoomNumber(null); @@ -1037,20 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService { if(StringUtils.isNotBlank(address.getTownship())){ formattedAddressBuilder.append(address.getTownship()); } + if(StringUtils.isNotBlank(building.getAddress())){ + Address orgAddr = new Address(); + orgAddr.setOrgAddress(building.getAddress()); + Address addr = AddressProcessors.process(orgAddr); + if(StringUtils.isBlank(address.getStreet())){ + address.setStreet(addr.getStreet()); + } + if(StringUtils.isBlank(address.getStreetNumber())){ + address.setStreetNumber(addr.getStreetNumber()); + } + } if(StringUtils.isNotBlank(address.getStreet())){ formattedAddressBuilder.append(address.getStreet()); } if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){ formattedAddressBuilder.append(address.getStreetNumber()); } - if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ - Address orgAddr = new Address(); - orgAddr.setOrgAddress(building.getAddress()); - Address addr = AddressProcessors.process(orgAddr); - building.getSearchAddress().setStreet(addr.getStreet()); - building.getSearchAddress().setStreetNumber(addr.getStreetNumber()); - formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber())); - } + if(StringUtils.isNotBlank(building.getSuggestName())){ formattedAddressBuilder.append(building.getSuggestName()); } @@ -1077,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ - formattedAddressBuilder.append(address.getRoomNumber()); -// formattedAddressBuilder.append(address.getRoomNumber() + "室"); +// formattedAddressBuilder.append(address.getRoomNumber()); + formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 58022cf03..1c8f2d004 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -77,6 +77,16 @@ public class Address { private String debugName; private AliyunBo aliyunBo; + //使用CRF切分的地址 + private String crfAddress; + + public String getCrfAddress() { + return crfAddress; + } + + public void setCrfAddress(String crfAddress) { + this.crfAddress = crfAddress; + } public AddressParserGoal getAddressParserGoal() { return addressParserGoal; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java index 498b12052..e53c9701f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java @@ -25,7 +25,15 @@ public class MatchParams { private boolean useMap; /**是否使用阿里云接口*/ private boolean useAliyunApi; + //是否使用CRF算法 + private boolean useCrf; + public boolean isUseCrf() { + return useCrf; + } + public void setUseCrf(boolean useCrf) { + this.useCrf = useCrf; + } public boolean isContainsLocation() { return containsLocation; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java index 93c69b4c4..ab5c929b3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java @@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor { @Override public RepeatStatus process(Address address) { String orgAddr = address.getEffectiveAddress(); + address.setCrfAddress(address.getAddress()); MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress()); if (matchResult != null) { String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index b6b22f5a1..bcbdba418 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -1,5 +1,7 @@ package com.fangjia.service.api; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse; import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest; import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse; import com.fangjia.core.util.JsonUtils; @@ -13,6 +15,9 @@ import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; +import java.util.ArrayList; +import java.util.List; + public class AliyunServiceImpl implements AliyunService { protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); @@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService { if(StringUtils.isNotBlank(city)){ request.setDefaultCity(city); } + AliyunBo aliyunBo = new AliyunBo(); try{ StructureAddressResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); String value = (String) responseJson.get("structure"); - AliyunBo aliyunBo = new AliyunBo(); for (String key:value.split("\t")){ String[] arry = key.split("="); wrapField(arry[0],arry[1], aliyunBo); } - return aliyunBo; }catch(Exception e){ - LOGGER.error("", e); - return null; + LOGGER.error("调用阿里标准化接口失败:", e); } + try{ + GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest(); + divisionRequest.setServiceCode("addrp"); + divisionRequest.setAppKey(APPKEY); + divisionRequest.setText(address); + GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest); + JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData()); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList<>(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + aliyunBo.setProv(array[0]); + aliyunBo.setCity(array[1]); + aliyunBo.setDistrict(array[2]); + aliyunBo.setTown(array[3]); + }catch(Exception e){ + LOGGER.error("调用阿里行政区划识别接口失败", e); + } + return aliyunBo; } private void wrapField(String field,String value, AliyunBo aliyunBo){ @@ -86,21 +115,38 @@ public class AliyunServiceImpl implements AliyunService { ACCESSKEYID, SECRET); IAcsClient client = new DefaultAcsClient(profile); - StructureAddressRequest request = new StructureAddressRequest(); +// StructureAddressRequest request = new StructureAddressRequest(); + GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest(); request.setServiceCode("addrp"); request.setAppKey(APPKEY); - request.setText("杭州市文一西路969号"); - request.setDefaultProvince("浙江省"); - request.setDefaultCity("杭州市"); - request.setDefaultDistrict("余杭区"); - StructureAddressResponse response = client.getAcsResponse(request); + request.setText("北郭乡东草亭村一号路2号"); +// request.setDefaultProvince("浙江省"); +// request.setDefaultCity("杭州市"); +// request.setDefaultDistrict("余杭区"); +// StructureAddressResponse response = client.getAcsResponse(request); + GetAddressDivisionCodeResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); - String value = (String) responseJson.get("structure"); - JSONObject json = new JSONObject(); + System.out.println(responseJson); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]); + /*JSONObject json = new JSONObject(); for (String key:value.split("\t")){ String[] arry = key.split("="); json.put(arry[0],arry[1]); } - System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class)); + System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/ } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 1e04da6c0..c236acbb4 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -2348,10 +2348,11 @@ public class AddressServiceImpl implements AddressService { /** * 基于高德开放接口的地址真实性检查 + * * @param city * @param address * @param checkBo - * @param addressList + * @param time */ private void checkAmapNew(String city, String address, AddressCheckBo checkBo, Set addressList) { try { @@ -3570,7 +3571,7 @@ public class AddressServiceImpl implements AddressService { * * @param city * @param address - * @param keywordsFormat + * @param checkBo */ private AddressCheckTruth checkTruthSplit(String city, String address, Map keywordsFormat) { // 1.拆分 @@ -4579,7 +4580,8 @@ public class AddressServiceImpl implements AddressService { /** * 地址评级具体逻辑 - * @param checkTr + * + * @param checkBo * @return */ private AddressCheckTruth gradeAddressTruth(AddressCheckTruth checkTr) { @@ -4602,8 +4604,8 @@ public class AddressServiceImpl implements AddressService { /** * 小区评分 - * @param check - * @param isConsume + * + * @param checkBo * @return */ private boolean gradeDistrictTruth(AddressCheckTruth check,boolean isConsume) { @@ -5130,7 +5132,7 @@ public class AddressServiceImpl implements AddressService { */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ String level = null; - LinkedList list = new LinkedList(Arrays.asList("province", "city", "district", "township", "street" + LinkedList list = new LinkedList(Arrays.asList("province", "city", "district", "township", "street", "streetNumber" , "name", "buildingNumber", "unitNumber", "roomNumber")); for(String key:list){ if(continueJudgeValue(json1, json2, key) == 3){