地址标准化修改

This commit is contained in:
xuhe
2021-05-09 17:39:25 +08:00
parent f69c8bd028
commit 4d4602d7ad
7 changed files with 222 additions and 48 deletions
@@ -1769,6 +1769,7 @@ public class AddressController {
// matchParams.setUseMap(true);
matchParams.setUseAliyunApi(true);
// matchParams.setUseCrf(true);
matchParams.setContainsLocation(model.isContainsLocation());
Building building = addressParserService.standard(matchParams);
Address address = building.getSearchAddress();
@@ -1,14 +1,12 @@
package com.fangjia.service.address;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.CNNMFilter;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.core.util.StringUtil;
import com.fangjia.core.util.*;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.AmapPoi;
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
import com.fangjia.dal.mongo.model.bkoff.Loudong;
import com.fangjia.dal.seaweed.dao.SeaweedDao;
@@ -25,6 +23,7 @@ import com.fangjia.service.mongo.bkoff.HouseLoudongService;
import com.fangjia.service.mongo.bkoff.LoudongService;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.bson.types.ObjectId;
@@ -134,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService {
formatAddress(building);
}
return building;
/*JSONObject result = getDataFromCRF("test");
return new Building();*/
}
@Override
@@ -165,6 +166,21 @@ public class AddressParserServiceImpl implements AddressParserService {
return building;
}
Building matchedBuilding;
//调用CRF算法切分数据
/*JSONObject jsonObj = null;
if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
jsonObj = getDataFromCRF(addr.getOrgAddress());
if(jsonObj != null){
String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") +
(jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") +
(jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : "");
if(StringUtils.isNotBlank(poi)){
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
addr.setKeywords(poi);
}
}
}
}*/
//keywords使用aliyun接口返回的poi
AliyunBo aliyunBo = null;
if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
@@ -176,6 +192,9 @@ public class AddressParserServiceImpl implements AddressParserService {
addr.setKeywords(poi);
}
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setCity(aliyunBo.getCity());
}
if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){
String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length()));
if(StringUtils.isNotBlank(str)){
@@ -187,26 +206,80 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setSuggestCity(aliyunBo.getCity());
}
}
}
AddressMatchMetaData meta = null;
if(StringUtils.isBlank(addr.getCity())){
addr.addKeyword(addr.getOrgAddress());
meta = getMapMeta(addr);
if(meta != null){
addr.setProvince(meta.getMapProvince());
addr.setCity(meta.getMapCity());
addr.setDistrict(meta.getDistrict());
}
}
addr.addKeyword(addr.getOrgAddress());
if(params.isUseMap()){
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP);
}else{
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB);
}
/*if(jsonObj != null){
//乡镇优化
String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : "";
if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){
String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township);
if(StringUtils.isNotBlank(town)){
addr.setTownship(town);
}
}
//道路优化(两边均去掉乡镇)
String street = addr.getStreet();
String streetNumber = addr.getStreetNumber();
String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : "";
String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : "";
//去掉路名中含有乡镇名的
if(StringUtils.isNotBlank(addr.getTownship())){
street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street;
streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf;
}
//crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理
if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){
//Java解析出路名为空
if(StringUtils.isBlank(street)){
street = streetCrf;
if(StringUtils.isNotBlank(streetNumberCrf)){
streetNumber = streetNumberCrf;
}
}else{
//CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则
if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){
street = streetCrf;
}
}
addr.setStreet(street);
addr.setStreetNumber(streetNumber);
}
//poi处理
//没有匹配上库中小区时
if(StringUtils.isBlank(matchedBuilding.getId())){
String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") +
(jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") +
(jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : "");
//去掉路名中含有乡镇名和路名 getParsedPropertyName
if(StringUtils.isNotBlank(addr.getTownship())){
poi = poi.replace(addr.getTownship(),"");
}
if(StringUtils.isNotBlank(addr.getStreet())){
poi = poi.replace(addr.getStreet(),"");
}
addr.setParsedPropertyName(poi);
}
//楼栋、单元、室号
String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", "");
addr.setOrgBuildingNumber(houseNo);
addr.setBuildingNumber(houseNo);
addr.setOrgUnitNumber(cellNo);
addr.setUnitNumber(cellNo);
addr.setFloor(floorNo);
addr.setOrgRoomNumber(roomNo);
addr.setRoomNumber(roomNo);
}*/
matchedBuilding.setProvince(addr.getProvince());
matchedBuilding.setSearchAddress(addr);
matchedBuilding.setSuggestName(getSuggestName(matchedBuilding));
@@ -215,7 +288,8 @@ public class AddressParserServiceImpl implements AddressParserService {
completBuildingField(matchedBuilding, aliyunBo, params);
}
Address tempAddr = matchedBuilding.getSearchAddress();
if(StringUtils.isBlank(tempAddr.getStreet()) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
AddressMatchMetaData meta = getMapMeta(addr);
if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){
matchedBuilding.setAddress(meta.getAddress());
}
@@ -255,6 +329,25 @@ public class AddressParserServiceImpl implements AddressParserService {
}
return null;
}
/**
* TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用
* @param address
* @return
*/
/*private JSONObject getDataFromCRF(String address){
HttpRequest request = new HttpRequest();
Map<String,String> params = new HashMap<>();
params.put("address",address);
try {
JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8");
System.out.println(resp);
return resp;
}catch(Exception e){
}
return null;
}*/
/**
* 信息不全时使用阿里接口补全数据
* @param matchedBuilding
@@ -271,28 +364,39 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
Address addr = matchedBuilding.getSearchAddress();
if(StringUtils.isBlank(addr.getProvince())){
if(StringUtils.isNotBlank(aliyunBo.getProv())){
addr.setProvince(formatProvince(aliyunBo.getProv()));
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isNotBlank(aliyunBo.getDistrict())){
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
}
addr.setTownship(aliyunBo.getTown());
/*if(StringUtils.isBlank(addr.getProvince())){
addr.setProvince(formatProvince(aliyunBo.getProv()));
}
if(StringUtils.isBlank(addr.getCity())){
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isBlank(addr.getDevelopmentZone())){
addr.setDevelopmentZone(aliyunBo.getDevzone());
}
if(StringUtils.isBlank(addr.getDistrict())){
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
}
if(StringUtils.isBlank(addr.getTownship())){
addr.setTownship(aliyunBo.getTown());
}*/
if(StringUtils.isBlank(addr.getDevelopmentZone())){
addr.setDevelopmentZone(aliyunBo.getDevzone());
}
/*addr.setOrgBuildingNumber(aliyunBo.getHouseno());
addr.setBuildingNumber(aliyunBo.getHouseno());
addr.setOrgUnitNumber(aliyunBo.getCellno());
addr.setUnitNumber(aliyunBo.getCellno());*/
if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){
addr.setRoomNumber(aliyunBo.getRoomno());
addr.setOrgRoomNumber(aliyunBo.getRoomno());
addr.setRoomNumber(aliyunBo.getRoomno());
}else{
addr.setRoomNumber(null);
@@ -1037,20 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService {
if(StringUtils.isNotBlank(address.getTownship())){
formattedAddressBuilder.append(address.getTownship());
}
if(StringUtils.isNotBlank(building.getAddress())){
Address orgAddr = new Address();
orgAddr.setOrgAddress(building.getAddress());
Address addr = AddressProcessors.process(orgAddr);
if(StringUtils.isBlank(address.getStreet())){
address.setStreet(addr.getStreet());
}
if(StringUtils.isBlank(address.getStreetNumber())){
address.setStreetNumber(addr.getStreetNumber());
}
}
if(StringUtils.isNotBlank(address.getStreet())){
formattedAddressBuilder.append(address.getStreet());
}
if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){
formattedAddressBuilder.append(address.getStreetNumber());
}
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
Address orgAddr = new Address();
orgAddr.setOrgAddress(building.getAddress());
Address addr = AddressProcessors.process(orgAddr);
building.getSearchAddress().setStreet(addr.getStreet());
building.getSearchAddress().setStreetNumber(addr.getStreetNumber());
formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber()));
}
if(StringUtils.isNotBlank(building.getSuggestName())){
formattedAddressBuilder.append(building.getSuggestName());
}
@@ -1077,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService {
formattedAddressBuilder.append(address.getUnitNumber() + "单元");
}
if(StringUtils.isNotBlank(address.getRoomNumber())){
formattedAddressBuilder.append(address.getRoomNumber());
// formattedAddressBuilder.append(address.getRoomNumber() + "室");
// formattedAddressBuilder.append(address.getRoomNumber());
formattedAddressBuilder.append(address.getRoomNumber() + "室");
}
address.setFormattedAddress(formattedAddressBuilder.toString());
@@ -77,6 +77,16 @@ public class Address {
private String debugName;
private AliyunBo aliyunBo;
//使用CRF切分的地址
private String crfAddress;
public String getCrfAddress() {
return crfAddress;
}
public void setCrfAddress(String crfAddress) {
this.crfAddress = crfAddress;
}
public AddressParserGoal getAddressParserGoal() {
return addressParserGoal;
@@ -25,7 +25,15 @@ public class MatchParams {
private boolean useMap;
/**是否使用阿里云接口*/
private boolean useAliyunApi;
//是否使用CRF算法
private boolean useCrf;
public boolean isUseCrf() {
return useCrf;
}
public void setUseCrf(boolean useCrf) {
this.useCrf = useCrf;
}
public boolean isContainsLocation() {
return containsLocation;
}
@@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor {
@Override
public RepeatStatus process(Address address) {
String orgAddr = address.getEffectiveAddress();
address.setCrfAddress(address.getAddress());
MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress());
if (matchResult != null) {
String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent);
@@ -1,5 +1,7 @@
package com.fangjia.service.api;
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest;
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse;
import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest;
import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse;
import com.fangjia.core.util.JsonUtils;
@@ -13,6 +15,9 @@ import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import java.util.ArrayList;
import java.util.List;
public class AliyunServiceImpl implements AliyunService {
protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass());
@@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService {
if(StringUtils.isNotBlank(city)){
request.setDefaultCity(city);
}
AliyunBo aliyunBo = new AliyunBo();
try{
StructureAddressResponse response = client.getAcsResponse(request);
JSONObject responseJson = JSONObject.fromObject(response.getData());
String value = (String) responseJson.get("structure");
AliyunBo aliyunBo = new AliyunBo();
for (String key:value.split("\t")){
String[] arry = key.split("=");
wrapField(arry[0],arry[1], aliyunBo);
}
return aliyunBo;
}catch(Exception e){
LOGGER.error("", e);
return null;
LOGGER.error("调用阿里标准化接口失败:", e);
}
try{
GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest();
divisionRequest.setServiceCode("addrp");
divisionRequest.setAppKey(APPKEY);
divisionRequest.setText(address);
GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest);
JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData());
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
aliyunBo.setProv(array[0]);
aliyunBo.setCity(array[1]);
aliyunBo.setDistrict(array[2]);
aliyunBo.setTown(array[3]);
}catch(Exception e){
LOGGER.error("调用阿里行政区划识别接口失败", e);
}
return aliyunBo;
}
private void wrapField(String field,String value, AliyunBo aliyunBo){
@@ -86,21 +115,38 @@ public class AliyunServiceImpl implements AliyunService {
ACCESSKEYID,
SECRET);
IAcsClient client = new DefaultAcsClient(profile);
StructureAddressRequest request = new StructureAddressRequest();
// StructureAddressRequest request = new StructureAddressRequest();
GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest();
request.setServiceCode("addrp");
request.setAppKey(APPKEY);
request.setText("杭州市文一西路969号");
request.setDefaultProvince("浙江省");
request.setDefaultCity("杭州市");
request.setDefaultDistrict("余杭区");
StructureAddressResponse response = client.getAcsResponse(request);
request.setText("北郭乡东草亭村一号路2号");
// request.setDefaultProvince("浙江省");
// request.setDefaultCity("杭州市");
// request.setDefaultDistrict("余杭区");
// StructureAddressResponse response = client.getAcsResponse(request);
GetAddressDivisionCodeResponse response = client.getAcsResponse(request);
JSONObject responseJson = JSONObject.fromObject(response.getData());
String value = (String) responseJson.get("structure");
JSONObject json = new JSONObject();
System.out.println(responseJson);
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<String>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
System.out.println(array[0]);
System.out.println(array[1]);
System.out.println(array[2]);
System.out.println(array[3]);
/*JSONObject json = new JSONObject();
for (String key:value.split("\t")){
String[] arry = key.split("=");
json.put(arry[0],arry[1]);
}
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/
}
}
@@ -5132,7 +5132,7 @@ public class AddressServiceImpl implements AddressService {
*/
private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){
String level = null;
LinkedList<String> list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street"
LinkedList<String> list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street", "streetNumber"
, "name", "buildingNumber", "unitNumber", "roomNumber"));
for(String key:list){
if(continueJudgeValue(json1, json2, key) == 3){