修改样本查询bug

This commit is contained in:
2022-06-15 22:20:43 +08:00
parent 5c520dd3e8
commit bcebf3fac6
3 changed files with 62 additions and 6 deletions
@@ -133,10 +133,9 @@ public abstract class AbstractAddressProcessor {
* @return
*/
protected String tokenDistrictName(String keyword){
List<Token> tokens = CnTokenizer.tokenizer(keyword, true);
Token token = getTokenByType(tokens, KeywordEnum.district);
if(token != null && "district".equals(token.getKeywordTypes()) && keyword.contains(token.getName())){
return token.getName();
List<String> tokens = tokenDistrictNames(keyword);
if(tokens.size() > 0){
return tokens.get(0);
}
return null;
}
@@ -159,7 +158,35 @@ public abstract class AbstractAddressProcessor {
protected List<String> tokenDistrictNames(String keyword){
List<String> names = new ArrayList<String>();
List<Token> tokens = CnTokenizer.tokenizer(keyword, true);
for (Token token : tokens) {
List<Token> mergeTokens = new ArrayList<Token>();
for(int i = 0; i < tokens.size(); i++){
Token token = tokens.get(i);
mergeTokens.add(token);
if(token.getKeywordTypes().equals("none") && i != tokens.size() - 1){
for(int j = i +1; j < tokens.size(); j++){
Token nextToken = tokens.get(j);
if("none".equals(nextToken.getKeywordTypes())){
token.setEnd(nextToken.getEnd());
token.setName(token.getName() + nextToken.getName());
i++;
}else{
break;
}
}
}else if(token.getKeywordTypes().equals("district") && i != tokens.size() - 1){
for(int j = i +1; j < tokens.size(); j++){
Token nextToken = tokens.get(j);
if("partition".equals(nextToken.getKeywordTypes()) || "stage".equals(nextToken.getKeywordTypes())){
token.setEnd(nextToken.getEnd());
token.setName(token.getName() + nextToken.getName());
i++;
}else{
break;
}
}
}
}
for (Token token : mergeTokens) {
if (token.containsType(KeywordEnum.district) && keyword.contains(token.getName())) {
names.add(token.getName());
}
@@ -19,7 +19,7 @@ public enum KeywordEnum {
channel, // 房价网频道
house_number, // 室号
stage, // 分期
partitions, // 分区
partition, // 分区
town,//乡镇
village,//居委会
number_cn, total_price, room, area, hall, toilet, alpha;
@@ -14,6 +14,8 @@ public class PatternTokenizer {
private static Pattern reg_house_number; // 匹配"5号301室"
private static Pattern REG_RANGE_PATT; // 匹配"5号301室"
private static Pattern REG_NUMBER_CN; // 匹配面积
private static Pattern STAGE_PATTERN;// 分期
private static Pattern PARTITION_PATTERN;// 分区
private static final String REG_NUM = "(((,|\\.|\\d| )*\\d)|(一|二|三|四|五|六|七|八|九|十|零|两|个|十|百|千|万|亿| )+)";
private static final String REG_RANGE_SPLITTER = "(-|~|~|-|到|至| )";
@@ -31,6 +33,8 @@ public class PatternTokenizer {
REG_RANGE_PATT = Pattern.compile(REG_RANGE_SPLITTER);
REG_NUMBER_CN = Pattern.compile(REG_NUM + "(弄|号|栋|厅|卫)");
STAGE_PATTERN = Pattern.compile("(([零一二三四五六七八九十百千\\d]+)期?[,、至到]?)?([零一二三四五六七八九十百千\\d]+)期");
PARTITION_PATTERN = Pattern.compile("(\\w|东|南|北|中|西|甲|乙|丙|丁|零|一|二|三|四|五|六|七|八|九|十)+(区|坊)");
}
public static void main(String[] args) {
@@ -52,6 +56,8 @@ public class PatternTokenizer {
}
public static String tokenizers(List<Token> tokens, String k) {
k = parsePartition(tokens, k);
k = parseStage(tokens, k);
// 面积
k = parseArea(tokens, k);
// 房型
@@ -77,6 +83,29 @@ public class PatternTokenizer {
}
return t;
}
private static String parseStage(List<Token> ret, String t) {
String value = null;
Matcher m = STAGE_PATTERN.matcher(t);
while (m.find()) {
value = m.group();
t = disableValue(t, value);
ret.add(new Token(value, m.start(), m.end(), KeywordEnum.stage.name()));
m = STAGE_PATTERN.matcher(t);
}
return t;
}
private static String parsePartition(List<Token> ret, String t) {
String value = null;
Matcher m = PARTITION_PATTERN.matcher(t);
while (m.find()) {
value = m.group();
t = disableValue(t, value);
ret.add(new Token(value, m.start(), m.end(), KeywordEnum.partition.name()));
m = PARTITION_PATTERN.matcher(t);
}
return t;
}
private static String parseArea(List<Token> ret, String t) {
String value = null;