Merge branch 'script_gj' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2025-03-18 17:00:40 +08:00
3 changed files with 65 additions and 34 deletions
@@ -1,12 +1,26 @@
package com.fangjia.core.util;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.dictionary.py.Pinyin;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
import net.sf.json.JSONArray;
import net.sf.json.JSONNull;
import net.sf.json.JSONObject;
import net.sourceforge.pinyin4j.PinyinHelper;
import net.sourceforge.pinyin4j.format.HanyuPinyinOutputFormat;
import net.sourceforge.pinyin4j.format.HanyuPinyinToneType;
import net.sourceforge.pinyin4j.format.HanyuPinyinVCharType;
import net.sourceforge.pinyin4j.format.exception.BadHanyuPinyinOutputFormatCombination;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sothis.core.util.CollectionUtils;
import java.io.*;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
@@ -23,28 +37,6 @@ import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import com.hankcs.hanlp.HanLP;
import com.hankcs.hanlp.dictionary.py.Pinyin;
import net.sf.json.JSONArray;
import net.sf.json.JSONNull;
import net.sf.json.JSONObject;
import net.sourceforge.pinyin4j.PinyinHelper;
import net.sourceforge.pinyin4j.format.HanyuPinyinOutputFormat;
import net.sourceforge.pinyin4j.format.HanyuPinyinToneType;
import net.sourceforge.pinyin4j.format.HanyuPinyinVCharType;
import net.sourceforge.pinyin4j.format.exception.BadHanyuPinyinOutputFormatCombination;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang.math.NumberUtils;
import org.apache.lucene.queryParser.QueryParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.sothis.core.util.CollectionUtils;
import com.mongodb.BasicDBList;
import com.mongodb.BasicDBObject;
public class DataUtil {
public static Pattern cellPhonePattern = Pattern.compile("^(1[23456789][0-9]{1})(-)?([0-9]{8})$");// 判断是否手机号码
private final static Logger LOGGER = LoggerFactory.getLogger(DataUtil.class);
@@ -1040,10 +1032,11 @@ public class DataUtil {
keyword = keyword.replace("\"", "");
keyword = keyword.replaceAll(SPACIAL_PATTERN, "");
keyword = keyword.replace("\\", " ");
keyword = QueryParser.escape(keyword.trim()).trim();
// keyword = QueryParser.escape(keyword.trim());
keyword = DataUtil.toDBCcase(keyword.replace(";", " "));
keyword = keyword.replaceAll(SPACIAL_PATTERN, "");
keyword = QueryParser.escape(keyword.trim()).trim();
// keyword = QueryParser.escape(keyword.trim()).trim();
keyword = keyword.trim();
for (String reservedWord : new String[] { "AND", "OR" }) {
if (keyword.contains(reservedWord)) {
keyword = keyword.replace(keyword, "\\" + keyword);
@@ -1063,10 +1056,11 @@ public class DataUtil {
s = s.replace("\"", "");
s = s.replaceAll(SPACIAL_PATTERN, "");
s = s.replace("\\", " ");
s = QueryParser.escape(s.trim()).trim();
// s = QueryParser.escape(s.trim()).trim();
s = DataUtil.toDBCcase(s.replace(";", " "));
s = s.replaceAll(SPACIAL_PATTERN, "");
s = QueryParser.escape(s.trim()).trim();
// s = QueryParser.escape(s.trim()).trim();
s = s.trim();
StringBuilder sb = new StringBuilder();
for (int i = 0; i < s.length(); i++) {
char c = s.charAt(i);
@@ -47,7 +47,7 @@ class DistrictDealSellRentRankBatch extends BaseBatch {
"图木舒克","五家渠","济源","仙桃","潜江","天门","景德镇", "神农架","五指山","琼海","儋州","文昌","万宁","东方","定安","屯昌","澄迈","临高","白沙","昌江","乐东","陵水",
"保亭","琼中","北屯","铁门关","双河","可克达拉"
]
cities = ["无锡"]
// cities = ["无锡"]
println cities
for(String city : cities) {
List<DBObject> districts = findDistrictByCity(city)
@@ -104,6 +104,43 @@ class FileUtils {
sb.setLength(0)
}
out.close();
return sb.toString().getBytes(charset)
}
static def createEncryptFile(fileName, outputStr, isAppend){
OutputStream out = new FileOutputStream(fileName, isAppend);
out.write(outputStr.getBytes("utf-8"));
out.close();
}
static def createFileStringByCustomSuffix(ret, fileName, charset, suffix, lineSuffix){
File file = new File(fileName);
// if (file.exists()) {
// file.delete();
// }
if (!file.getParentFile().exists()) {
boolean result = file.getParentFile().mkdirs();
if (!result) {
System.out.println("´´½¨Ä¿Â¼Ê§°Ü");
}
}
def sb = new StringBuffer();
def index = 0
for(def row : ret){
if(row){
index++
sb.append(StringUtils.join(row.toArray(), suffix))
if(index<ret.size()){
sb.append(lineSuffix+"\n");
}else{
sb.append(lineSuffix);
}
}
}
return sb.toString()
}