diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 46545fc04..1d5947440 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -759,6 +759,13 @@ public class AddressParserServiceImpl implements AddressParserService { double simScore; boolean onSameRoad = false; boolean sameDistrict = isSameDistrict(address, metaData); + if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){ + return 0; + } + //当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区 + if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){ + return 0; + } Iterator> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator(); Map.Entry addrEntry; while(addrEntrySet.hasNext()){ @@ -808,7 +815,6 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - String keywordPinYin = address.getKeywordsPinYin(); String keywords = address.getKeywords(); String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin(); @@ -904,23 +910,25 @@ public class AddressParserServiceImpl implements AddressParserService { double matchLevelScore = 0; double similarityScore = 0; - if (standardAddressScore > 0 && keywordScore > 0) { + if(metaData.getId().equals(address.getPolygonSeaweedId())){ matchLevelScore = 0.9; - similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; - } else if(keywordScore > 0 && streetScore > 0){ + similarityScore = 0.05; + }else if (standardAddressScore > 0 && keywordScore > 0) { matchLevelScore = 0.8; - similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; + similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE; } else if (standardAddressScore > 0) { matchLevelScore = 0.7; similarityScore = standardAddressScore * SIMILARITY_RATE; + } else if(keywordScore > 0 && streetScore > 0){ + matchLevelScore = 0.6; + similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE; } else if (keywordScore > 0) { matchLevelScore = 0.6; similarityScore = keywordScore * SIMILARITY_RATE; } else if (fuzzyKeywordsScore > 0) { - matchLevelScore = 0.5; + matchLevelScore = 0.4; similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE; } - if (similarityScore > 0) { if(metaData.getBuildingType() != null){ if ("shop".equals(metaData.getBuildingType().toString())) { @@ -942,10 +950,6 @@ public class AddressParserServiceImpl implements AddressParserService { * similarityScore = 相似度部分(0.07) + 类型部分(0.01) < 0.1 * */ double score = matchLevelScore + similarityScore; - if(address.hasCoordinate() && metaData.hasCoordinate() - && GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){ - score = 0d; - } return score; } @@ -980,7 +984,7 @@ public class AddressParserServiceImpl implements AddressParserService { return false; } if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){ - return false; + return true; } String district = metaData.getDistrict(); return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 720da51bb..ed723c1d5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -61,6 +61,8 @@ public class Address { private String suggestDistrict; private String suggestName; private String suggestNamePinYin; + private String polygonSeaweedId; + private String geoRegion; private List buildingNumbers; private PreciseLevel preciseLevel; private boolean pinYin; @@ -750,6 +752,14 @@ public class Address { return false; } + public String getGeoRegion() { + return geoRegion; + } + + public void setGeoRegion(String geoRegion) { + this.geoRegion = geoRegion; + } + @Override public String toString() { return "Address{" + @@ -809,6 +819,14 @@ public class Address { '}'; } + public String getPolygonSeaweedId() { + return polygonSeaweedId; + } + + public void setPolygonSeaweedId(String polygonSeaweedId) { + this.polygonSeaweedId = polygonSeaweedId; + } + public String toMd5Hex(){ String keyProperties = "Address{" + "city='" + city + '\'' + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java index aa72e6d01..9097a3a63 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/AddressProcessors.java @@ -20,11 +20,9 @@ public class AddressProcessors{ static { //修改顺序需要调整process i 的值 - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new ProximalCharactersProcessor()); addressProcessors.add(new PreNoiseProcessor()); addressProcessors.add(new AddressFormatProcessor()); - addressProcessors.add(new TokenProcessor()); addressProcessors.add(new FloorProcessor()); addressProcessors.add(new TooltipProcessor()); addressProcessors.add(new ProvinceProcessor()); @@ -44,7 +42,7 @@ public class AddressProcessors{ addressProcessors.add(new NumericProcessor()); addressProcessors.add(new ProvinceAdjustProcessor()); addressProcessors.add(new FormatProcessor()); -// addressProcessors.add(new GeoProcessor()); + addressProcessors.add(new GeoProcessor()); } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 6ae746ab5..130b7eea9 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -52,6 +52,7 @@ public class CityProcessor extends AbstractAddressProcessor { } if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){ String formatCity = RegionData.formatCity(city); + address.setCity(formatCity); String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null; String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity; /**如果省份和城市信息相互矛盾 则用区域推断城市*/ @@ -191,7 +192,6 @@ public class CityProcessor extends AbstractAddressProcessor { }else{ weight.score += 2; } - }else if(address.contains(city)){ index = address.indexOf(city); weight.index = index; @@ -208,9 +208,11 @@ public class CityProcessor extends AbstractAddressProcessor { if(matchResult !=null){ address = address.replace(matchResult.matchContent, ""); } - if(address.indexOf(region, index) > -1){ + if(address.indexOf(region + "区", index) > -1 || address.indexOf(region + "县", index) > -1){ if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){ weight.score += 2; + }else if(city.equals(addr.getCity())){ + weight.score += 10; }else{ weight.score += 1; } @@ -223,7 +225,11 @@ public class CityProcessor extends AbstractAddressProcessor { for(String alia : alias){ if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1 && (alia.endsWith("市")|(alia.endsWith("县")))){ - weight.score += 1; + if(city.equals(addr.getCity())){ + weight.score += 10; + }else{ + weight.score += 1; + } weight.index = address.indexOf(alia, index); break; } @@ -237,16 +243,6 @@ public class CityProcessor extends AbstractAddressProcessor { } List weights = new ArrayList(weightMap.values()); if(weights.size() > 0){ - /*String street = getStreet(address); - if(street != null){ - Iterator iterator = weights.iterator(); - while(iterator.hasNext()){ - FragmentWeight fragmentWeight = iterator.next(); - if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){ - iterator.remove(); - } - } - }*/ Collections.sort(weights); } if(weights.size() > 0){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java index 5cfba96b5..697e69f8d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java @@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor { address.setDevelopmentZone(dz); } } + if(address.getSuggestDistrict() != null && address.getDistrict() == null){ + address.setDistrict(address.getSuggestDistrict()); + } address.setAddress(orgAddress); address.setEffectiveAddress(address.getAddress()); return RepeatStatus.NEXT; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java index 83d74a0d4..c72141550 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/GeoProcessor.java @@ -5,8 +5,11 @@ import com.fangjia.core.util.HttpRequest; import com.fangjia.service.address.mode.Address; import net.sf.json.JSONArray; import net.sf.json.JSONObject; -import org.apache.commons.lang.StringUtils; -import java.util.*; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; /** * 地址经纬度处理器 @@ -18,7 +21,6 @@ public class GeoProcessor extends AbstractAddressProcessor { private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde"; List POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); - List DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口"); @Override public RepeatStatus process(Address address) { @@ -26,68 +28,55 @@ public class GeoProcessor extends AbstractAddressProcessor { if (address.hasCoordinate()) { return RepeatStatus.NEXT; } - List queryTypes = new ArrayList(); - List addressList = new ArrayList(); - if (address.isStandardAddress()) { - addressList.add(address.getStandardAddress()); - queryTypes.add("StandardAddress"); - } - if (StringUtils.isNotBlank(address.getOrgKeywords())) { - addressList.add(address.getOrgKeywords()); - queryTypes.add("Keywords"); - } HttpRequest request = new HttpRequest(); Map params = new HashMap(); - params.put("address", StringUtils.join(addressList, "|")); + params.put("address", address.getOrgAddress()); params.put("key", KEY); params.put("city", address.getCity()); - params.put("batch", "true"); params.put("output", "json"); try { JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8"); if (response.getInt("status") == 1) { JSONArray geocodes = response.getJSONArray("geocodes"); - if(queryTypes.contains("StandardAddress")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); + if(geocodes.size() > 0){ + JSONObject json = geocodes.getJSONObject(0); + if(json.get("district") instanceof String){ + String city = address.getCity(); + if(city == null){ + city = RegionData.formatCity(json.getString("city")); } + String region = RegionData.formatRegion(city, json.getString("district")); + address.setGeoRegion(region); } - if(geocode.containsKey("street") && geocode.containsKey("number")){ - String geocodeAddress = geocode.getString("street") + geocode.getString("number"); - if(address.getStandardAddress().contains(geocodeAddress)){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); - } - } - } - if(queryTypes.contains("Keywords")){ - JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords")); - String level = geocode.getString("level"); - if(DISTRICT_LEVEL.contains(level)){ - if(StringUtils.isBlank(address.getDistrict())){ - address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district"))); - } - } - if(POI_LEVELS.contains(level)){ - String formattedAddress = geocode.getString("formatted_address"); - formattedAddress = formattedAddress - .replaceFirst(geocode.getString("province"), "") - .replaceFirst(geocode.getString("city"), "") - .replaceFirst(geocode.getString("district"), ""); - if(formattedAddress.equals(address.getOrgKeywords())){ - String location = geocode.getString("location"); - address.setLng(Double.parseDouble(location.split(",")[0])); - address.setLat(Double.parseDouble(location.split(",")[1])); + if(POI_LEVELS.contains(json.getString("level"))){ + //对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼 + if("门牌号".equals(json.getString("level"))){ + if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){ + return RepeatStatus.NEXT; + } } + String location = json.getString("location"); + address.setLng(Double.valueOf(location.split(",")[0])); + address.setLat(Double.valueOf(location.split(",")[1])); + String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}"; + tmp = tmp.replace("location", location); + Map headers = new HashMap(); + headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz"); + JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8")); + if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){ + JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits"); + if(hits.size() > 0){ + String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id"); + address.setPolygonSeaweedId(seaweedId); + } + } + } } + } } catch (Exception e) { - + e.printStackTrace(); } return RepeatStatus.NEXT; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java index 5236b7d55..797d9e367 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/VillageNameProcessor.java @@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor { @Override public AbstractAddressProcessor.RepeatStatus process(Address address) { String orgAddress = address.getAddress(); - if(address.getTokens()!=null){ - for(Token token:address.getTokens()){ - if("district".equals(token.getKeywordTypes())){ - address.setKeywords(token.getName()); - address.setAddress(orgAddress.replace(token.getName(),"")); - return RepeatStatus.NEXT; - } - } - } List names = tokenDistrictNames(orgAddress); List kws = new ArrayList(); for(String name : names){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java index de010e201..c216a1eba 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/cnAnalyzer/CnTokenizer.java @@ -1,24 +1,16 @@ package com.fangjia.service.cnAnalyzer; -import java.io.IOException; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; -import org.apache.solr.client.solrj.util.ClientUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - import com.fangjia.core.util.DataUtil; import com.goojia.solr.ganalytics.IKSegmentation; import com.goojia.solr.ganalytics.Lexeme; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.io.StringReader; +import java.io.UnsupportedEncodingException; +import java.util.*; /** * 房价网分词器