添加围栏,限制匹配小区区域

This commit is contained in:
wangwenyan@fangjia.com
2022-05-18 15:45:01 +08:00
parent 71eedee536
commit 6af5da836c
8 changed files with 93 additions and 102 deletions
@@ -759,6 +759,13 @@ public class AddressParserServiceImpl implements AddressParserService {
double simScore;
boolean onSameRoad = false;
boolean sameDistrict = isSameDistrict(address, metaData);
if(PreciseLevel.STRICT == address.getPreciseLevel() && !sameDistrict){
return 0;
}
//当地图解析出来的区域和用户地址中的区域一致时,只匹配指定区域下面的小区
if(address.getGeoRegion() != null && address.getGeoRegion().equals(address.getDistrict()) && !sameDistrict){
return 0;
}
Iterator<Map.Entry<String, MatchText>> addrEntrySet = metaData.getAddrePinYinMap().entrySet().iterator();
Map.Entry<String, MatchText> addrEntry;
while(addrEntrySet.hasNext()){
@@ -808,7 +815,6 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
}
String keywordPinYin = address.getKeywordsPinYin();
String keywords = address.getKeywords();
String fuzzyKeywordsPinYin = address.getFuzzyKeywordsPinYin();
@@ -904,23 +910,25 @@ public class AddressParserServiceImpl implements AddressParserService {
double matchLevelScore = 0;
double similarityScore = 0;
if (standardAddressScore > 0 && keywordScore > 0) {
if(metaData.getId().equals(address.getPolygonSeaweedId())){
matchLevelScore = 0.9;
similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE;
} else if(keywordScore > 0 && streetScore > 0){
similarityScore = 0.05;
}else if (standardAddressScore > 0 && keywordScore > 0) {
matchLevelScore = 0.8;
similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
similarityScore = Math.max(standardAddressScore, keywordScore) * SIMILARITY_RATE;
} else if (standardAddressScore > 0) {
matchLevelScore = 0.7;
similarityScore = standardAddressScore * SIMILARITY_RATE;
} else if(keywordScore > 0 && streetScore > 0){
matchLevelScore = 0.6;
similarityScore = Math.max(streetScore, keywordScore * 0.8) * SIMILARITY_RATE;
} else if (keywordScore > 0) {
matchLevelScore = 0.6;
similarityScore = keywordScore * SIMILARITY_RATE;
} else if (fuzzyKeywordsScore > 0) {
matchLevelScore = 0.5;
matchLevelScore = 0.4;
similarityScore = fuzzyKeywordsScore * SIMILARITY_RATE;
}
if (similarityScore > 0) {
if(metaData.getBuildingType() != null){
if ("shop".equals(metaData.getBuildingType().toString())) {
@@ -942,10 +950,6 @@ public class AddressParserServiceImpl implements AddressParserService {
* similarityScore = 相似度部分(0.07) + 类型部分(0.01) < 0.1
* */
double score = matchLevelScore + similarityScore;
if(address.hasCoordinate() && metaData.hasCoordinate()
&& GeoUtils.calculateDistance(address.getLng(), address.getLat(), metaData.getLng(), metaData.getLat()) > 3 * 1000){
score = 0d;
}
return score;
}
@@ -980,7 +984,7 @@ public class AddressParserServiceImpl implements AddressParserService {
return false;
}
if(StringUtils.isBlank(address.getDistrict()) && StringUtils.isBlank(address.getSuggestDistrict())){
return false;
return true;
}
String district = metaData.getDistrict();
return district.equals(address.getDistrict()) || district.equals(address.getSuggestDistrict());
@@ -61,6 +61,8 @@ public class Address {
private String suggestDistrict;
private String suggestName;
private String suggestNamePinYin;
private String polygonSeaweedId;
private String geoRegion;
private List<String> buildingNumbers;
private PreciseLevel preciseLevel;
private boolean pinYin;
@@ -750,6 +752,14 @@ public class Address {
return false;
}
public String getGeoRegion() {
return geoRegion;
}
public void setGeoRegion(String geoRegion) {
this.geoRegion = geoRegion;
}
@Override
public String toString() {
return "Address{" +
@@ -809,6 +819,14 @@ public class Address {
'}';
}
public String getPolygonSeaweedId() {
return polygonSeaweedId;
}
public void setPolygonSeaweedId(String polygonSeaweedId) {
this.polygonSeaweedId = polygonSeaweedId;
}
public String toMd5Hex(){
String keyProperties = "Address{" +
"city='" + city + '\'' +
@@ -20,11 +20,9 @@ public class AddressProcessors{
static {
//修改顺序需要调整process i 的值
addressProcessors.add(new TokenProcessor());
addressProcessors.add(new ProximalCharactersProcessor());
addressProcessors.add(new PreNoiseProcessor());
addressProcessors.add(new AddressFormatProcessor());
addressProcessors.add(new TokenProcessor());
addressProcessors.add(new FloorProcessor());
addressProcessors.add(new TooltipProcessor());
addressProcessors.add(new ProvinceProcessor());
@@ -44,7 +42,7 @@ public class AddressProcessors{
addressProcessors.add(new NumericProcessor());
addressProcessors.add(new ProvinceAdjustProcessor());
addressProcessors.add(new FormatProcessor());
// addressProcessors.add(new GeoProcessor());
addressProcessors.add(new GeoProcessor());
}
@@ -52,6 +52,7 @@ public class CityProcessor extends AbstractAddressProcessor {
}
if(StringUtils.isNotBlank(city) && CityStatus.SURE != address.getCityStatus() ){
String formatCity = RegionData.formatCity(city);
address.setCity(formatCity);
String possibleCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null ? parseCityFromAddress(orgAddress, address) : null;
String confirmCity = AddressParserGoal.Match.equals(address.getAddressParserGoal()) || address.getAddressParserGoal() == null? null : formatCity;
/**如果省份和城市信息相互矛盾 则用区域推断城市*/
@@ -191,7 +192,6 @@ public class CityProcessor extends AbstractAddressProcessor {
}else{
weight.score += 2;
}
}else if(address.contains(city)){
index = address.indexOf(city);
weight.index = index;
@@ -208,9 +208,11 @@ public class CityProcessor extends AbstractAddressProcessor {
if(matchResult !=null){
address = address.replace(matchResult.matchContent, "");
}
if(address.indexOf(region, index) > -1){
if(address.indexOf(region + "区", index) > -1 || address.indexOf(region + "县", index) > -1){
if(province != null && (orgAddress.contains(province + region) || orgAddress.contains(ProvinceData.getDisplay(province) + region))){
weight.score += 2;
}else if(city.equals(addr.getCity())){
weight.score += 10;
}else{
weight.score += 1;
}
@@ -223,7 +225,11 @@ public class CityProcessor extends AbstractAddressProcessor {
for(String alia : alias){
if(StringUtils.isNotBlank(alia) && !alia.equals(region) && address.indexOf(alia, index) > -1
&& (alia.endsWith("市")|(alia.endsWith("县")))){
weight.score += 1;
if(city.equals(addr.getCity())){
weight.score += 10;
}else{
weight.score += 1;
}
weight.index = address.indexOf(alia, index);
break;
}
@@ -237,16 +243,6 @@ public class CityProcessor extends AbstractAddressProcessor {
}
List<FragmentWeight> weights = new ArrayList<FragmentWeight>(weightMap.values());
if(weights.size() > 0){
/*String street = getStreet(address);
if(street != null){
Iterator<FragmentWeight> iterator = weights.iterator();
while(iterator.hasNext()){
FragmentWeight fragmentWeight = iterator.next();
if(match(street, ".*" + fragmentWeight.text + "[东南西北中一二三四五六七八九十百千\\d]?(路|街|大道|支路|村)")){
iterator.remove();
}
}
}*/
Collections.sort(weights);
}
if(weights.size() > 0){
@@ -167,6 +167,9 @@ public class DistrictProcessor extends AbstractAddressProcessor {
address.setDevelopmentZone(dz);
}
}
if(address.getSuggestDistrict() != null && address.getDistrict() == null){
address.setDistrict(address.getSuggestDistrict());
}
address.setAddress(orgAddress);
address.setEffectiveAddress(address.getAddress());
return RepeatStatus.NEXT;
@@ -5,8 +5,11 @@ import com.fangjia.core.util.HttpRequest;
import com.fangjia.service.address.mode.Address;
import net.sf.json.JSONArray;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import java.util.*;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* 地址经纬度处理器
@@ -18,7 +21,6 @@ public class GeoProcessor extends AbstractAddressProcessor {
private static final String KEY = "4ccfe3c898a9489884bd2bcb1ef6dcde";
List<String> POI_LEVELS = Arrays.asList("村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口");
List<String> DISTRICT_LEVEL = Arrays.asList("区县", "开发区", "乡镇", "村庄", "热点商圈", "兴趣点", "门牌号", "单元号", "道路交叉路口");
@Override
public RepeatStatus process(Address address) {
@@ -26,68 +28,55 @@ public class GeoProcessor extends AbstractAddressProcessor {
if (address.hasCoordinate()) {
return RepeatStatus.NEXT;
}
List<String> queryTypes = new ArrayList<String>();
List<String> addressList = new ArrayList<String>();
if (address.isStandardAddress()) {
addressList.add(address.getStandardAddress());
queryTypes.add("StandardAddress");
}
if (StringUtils.isNotBlank(address.getOrgKeywords())) {
addressList.add(address.getOrgKeywords());
queryTypes.add("Keywords");
}
HttpRequest request = new HttpRequest();
Map<String, String> params = new HashMap<String, String>();
params.put("address", StringUtils.join(addressList, "|"));
params.put("address", address.getOrgAddress());
params.put("key", KEY);
params.put("city", address.getCity());
params.put("batch", "true");
params.put("output", "json");
try {
JSONObject response = request.getJsonContent("https://restapi.amap.com/v3/geocode/geo", params, "UTF-8");
if (response.getInt("status") == 1) {
JSONArray geocodes = response.getJSONArray("geocodes");
if(queryTypes.contains("StandardAddress")){
JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("StandardAddress"));
String level = geocode.getString("level");
if(DISTRICT_LEVEL.contains(level)){
if(StringUtils.isBlank(address.getDistrict())){
address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district")));
if(geocodes.size() > 0){
JSONObject json = geocodes.getJSONObject(0);
if(json.get("district") instanceof String){
String city = address.getCity();
if(city == null){
city = RegionData.formatCity(json.getString("city"));
}
String region = RegionData.formatRegion(city, json.getString("district"));
address.setGeoRegion(region);
}
if(geocode.containsKey("street") && geocode.containsKey("number")){
String geocodeAddress = geocode.getString("street") + geocode.getString("number");
if(address.getStandardAddress().contains(geocodeAddress)){
String location = geocode.getString("location");
address.setLng(Double.parseDouble(location.split(",")[0]));
address.setLat(Double.parseDouble(location.split(",")[1]));
}
}
}
if(queryTypes.contains("Keywords")){
JSONObject geocode = geocodes.getJSONObject(queryTypes.indexOf("Keywords"));
String level = geocode.getString("level");
if(DISTRICT_LEVEL.contains(level)){
if(StringUtils.isBlank(address.getDistrict())){
address.setDistrict(RegionData.formatRegion(address.getCity(), geocode.getString("district")));
}
}
if(POI_LEVELS.contains(level)){
String formattedAddress = geocode.getString("formatted_address");
formattedAddress = formattedAddress
.replaceFirst(geocode.getString("province"), "")
.replaceFirst(geocode.getString("city"), "")
.replaceFirst(geocode.getString("district"), "");
if(formattedAddress.equals(address.getOrgKeywords())){
String location = geocode.getString("location");
address.setLng(Double.parseDouble(location.split(",")[0]));
address.setLat(Double.parseDouble(location.split(",")[1]));
if(POI_LEVELS.contains(json.getString("level"))){
//对于门牌号级别的POI,formatted_address中包含|代表是推测出来的路号,重庆市巴南区凤林路|808号,|号楼
if("门牌号".equals(json.getString("level"))){
if(!(json.get("number") instanceof JSONArray) && json.getString("formatted_address").contains("|" + json.getString("number"))){
return RepeatStatus.NEXT;
}
}
String location = json.getString("location");
address.setLng(Double.valueOf(location.split(",")[0]));
address.setLat(Double.valueOf(location.split(",")[1]));
String tmp = "{ \"_source\": [\"seaweed_id\"], \"query\": {\"geo_shape\": {\"shape\": {\"shape\": {\"type\": \"Point\",\"coordinates\":[location]},\"relation\": \"CONTAINS\"}}}}";
tmp = tmp.replace("location", location);
Map<String, String> headers = new HashMap<String, String>();
headers.put("Authorization","Basic ZGV2ZWxvcGVyOjVlOGVkZDg1MWQyZmRmYmQ3NDE1MjMyYzY3MzY3Y2Mz");
JSONObject resp = JSONObject.fromObject(request.postWithBody("http://192.168.0.206:9200/seaweed_polygon/_search", JSONObject.fromObject(tmp),headers,"UTF-8"));
if(resp.containsKey("hits") && resp.getJSONObject("hits").containsKey("hits")){
JSONArray hits = resp.getJSONObject("hits").getJSONArray("hits");
if(hits.size() > 0){
String seaweedId = hits.getJSONObject(0).getJSONObject("_source").getString("seaweed_id");
address.setPolygonSeaweedId(seaweedId);
}
}
}
}
}
} catch (Exception e) {
e.printStackTrace();
}
return RepeatStatus.NEXT;
}
@@ -15,15 +15,6 @@ public class VillageNameProcessor extends AbstractAddressProcessor {
@Override
public AbstractAddressProcessor.RepeatStatus process(Address address) {
String orgAddress = address.getAddress();
if(address.getTokens()!=null){
for(Token token:address.getTokens()){
if("district".equals(token.getKeywordTypes())){
address.setKeywords(token.getName());
address.setAddress(orgAddress.replace(token.getName(),""));
return RepeatStatus.NEXT;
}
}
}
List<String> names = tokenDistrictNames(orgAddress);
List<String> kws = new ArrayList<String>();
for(String name : names){
@@ -1,24 +1,16 @@
package com.fangjia.service.cnAnalyzer;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang.StringUtils;
import org.apache.solr.client.solrj.util.ClientUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.fangjia.core.util.DataUtil;
import com.goojia.solr.ganalytics.IKSegmentation;
import com.goojia.solr.ganalytics.Lexeme;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.StringReader;
import java.io.UnsupportedEncodingException;
import java.util.*;
/**
* 房价网分词器