diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 93218b265..c681c113d 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1435,6 +1435,25 @@ public class DataUtil { return null; } + /** + * 匹配符合条件的字符串 + * + * @param str + * @param pattern + * @return + */ + public static List matchAll(String str, Pattern pattern) { + if (str == null) { + return null; + } + List list = new ArrayList(); + Matcher matcher = pattern.matcher(str); + while (matcher.find()) { + list.add(matcher.group()); + } + return list; + } + public static boolean isNumeric(String str) { try { Double.parseDouble(str); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy new file mode 100644 index 000000000..79cf910db --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang.StringUtils + +/** + * 补充详细地址中缺失的乡镇街道与居委社区 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", { + List item -> + def area = item['F'] + def township = item['G'] + def community = item['H'] + String detailAddr = item['I'] + + // 不包含街道与居委 + if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && !detailAddr.contains(community) ) { + if (StringUtils.isNotBlank(area)) { + int index = detailAddr.indexOf(area + "区") + area.length() + 2 + item['K'] = detailAddr.substring(0, index) + township + community + + detailAddr.substring(index) + } + // 不包含社区 + } else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + detailAddr.contains(township) && !detailAddr.contains(community)) { + int index = detailAddr.indexOf(township) + township.length() + 1 + item['K'] = detailAddr.substring(0, index) + community + + detailAddr.substring(index) + } // 只不包含街道 + else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && detailAddr.contains(community)) { + if (StringUtils.isNotBlank(area)) { + int index = detailAddr.indexOf(area + "区") + area.length() + 2 + item['K'] = detailAddr.substring(0, index) + township + + detailAddr.substring(index) + } + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy new file mode 100644 index 000000000..b77b6d36c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +import java.util.regex.Pattern + +//ExcelUtils.removeEmptyRow("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V10(优化补充社区冗余信息).xlsx") +ners = ["houseno", "cellno", "floorno", "roomno"] +Pattern GROUP_PATTERN = Pattern.compile("[一二三四五六七八九十甲乙丙丁\\d]+组") +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx",{List item-> + + if (StringUtils.isNotBlank(item["J"])) { + //item["I"] = item["J"] + //item["W"] = "详细地址替换" + } +// def addr = item["B"].toString() +// def detailAddr = item["I"]?.toString() +// def pd = item["C"] +// item['J'] = "" +// if (StringUtils.isNotBlank(pd)) { +// int len = addr.lastIndexOf(pd) +// if (len > 3) { +// String aim = addr.substring(len - 3, len) +// int aimIndex = detailAddr.lastIndexOf(aim) +// if (aimIndex > 0) { +// item['J'] = detailAddr.substring(0, aimIndex + 3) + pd +// } +// } +// } + +// if(detailAddr{ +// String group = DataUtil.match(pd, GROUP_PATTERN) +// if(group == null ){ +// return +// } +// def arabic_group = null +// if(!detailAddr.contains(group)){ +// arabic_group = CNNMFilter.qCNNumericToArabic(group) +// } +// if(group){ +// def flg = addr.substring(addr.indexOf(group) - 3, addr.indexOf(group)) +// flg = flg.replace("-", "") +// if(arabic_group != null){ +// group = arabic_group +// } +// if(!detailAddr.contains(flg + group)){ +// println(item) +// detailAddr = detailAddr.replace(group, '') +// def idx = detailAddr.lastIndexOf(flg) +// try{ +// detailAddr = detailAddr.substring(0, idx) + flg + group + detailAddr.substring(idx + flg.length()) +// item["j"] = detailAddr +// }catch(Exception e){ +// item["W"] ="组替换失败" +// } +// +// +// } +// } +// } + +// HttpRequest request = new HttpRequest() +// JSONObject result = request.getJsonContent("http://192.168.0.19:31080/FangJiaNlpService/ner",["address":addr], "utf-8") +// def index = Integer.MAX_VALUE +// for(JSONObject entity : result.entities){ +// if(entity.type in ners){ +// if(entity.start < index){ +// index = entity.start +// } +// } +// } +// if(index != Integer.MAX_VALUE){ +// item["C"] = addr.substring(index) +// println(item) +// } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy new file mode 100644 index 000000000..f466ed3be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取省市区编码 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def province = item['C'] + def city = item['D'] + def area = item['E'] + if (StringUtils.isBlank(province) && city == '重庆' && item['K'] == null) { + item['J'] = '5000000' + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area)) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['J'] = obj.province_code + item['K'] = obj.area_code + } + } + } else if (StringUtils.isNotBlank(province) && StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area) && item['K'] == null) { + villagesConn.find(new BasicDBObject("province_name", new BasicDBObject('$regex', '^' + province)) + .append("city_name", new BasicDBObject('$regex', '^' + city)) + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['I'] = obj.province_code + item['J'] = obj.city_code + item['K'] = obj.area_code + } + } + if (item['I'] != null) { + item['I'] = (String.valueOf(item['I']).replace(".0", "") + "000000").substring(0, 6); + } + if (item['J'] != null) { + item['J'] = (String.valueOf(item['J']).replace(".0", "") + "000000").substring(0, 6); + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy new file mode 100644 index 000000000..8125bf1db --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + // 乡镇以下级政府及事业单位(居委会) + def types = "130106" + def city = item['D'] + def lng = item['M'] + def lat = item['N'] + if (StringUtils.isBlank(item['G'])) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + AmapPoi amapPoi = pois.get(0) + if (amapPoi.getCityname().startsWith(city)) { + item['G'] = amapPoi.getName() + item['S'] = '修改社区' + } + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy new file mode 100644 index 000000000..6b37becf8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy @@ -0,0 +1,17 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + if (item['L'] != null) { + def location = GeoUtils.gcjToGps84(item['L'], item['K']) + if (location) { + item['N'] = location.lat + item['M'] = location.lng + println(item) + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy new file mode 100644 index 000000000..c8481a44a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取国家标准居委街道 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def flag = item['S'] + def area = item['E'] + def name = item['G'] + if (StringUtils.isNotBlank(item['G']) && StringUtils.isNotBlank(area) && String && flag == '修改社区' && item['T'] == null) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", area + "区").append("name", name), new BasicDBObject()).limit(1).each { + obj -> + item['F'] = obj.street_name + item['L'] = obj.street_code + item['T'] = "补充街道正确" + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy index 7c92e5a3e..6994cf1dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy @@ -8,7 +8,7 @@ import com.fangjia.script.xingyebank.CityUtil import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据test.xlsx",{List item-> +ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据.xlsx",{List item-> if(StringUtils.isBlank(item['B'])){ return; @@ -20,6 +20,11 @@ ExcelUtils.multiThreadingModify("D:\\ params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" JSONObject json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + if((!json || !json.result) && StringUtils.isNotBlank(item['G'])){ + params["address"] = item['G'] + json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + } + if(json.code == 200 && json.result){ JSONObject result = json.result item['C'] = CityUtil.getProvinceByCity(result.city) @@ -28,8 +33,6 @@ ExcelUtils.multiThreadingModify("D:\\ item['F'] = result.name if (result.name) { item['O'] = '住宅地址' - } else { - item['O'] = '非住宅地址' } item['G'] = result.address item['H'] = result.lng @@ -51,6 +54,9 @@ ExcelUtils.multiThreadingModify("D:\\ item['J'] = json.getJSONObject("result").averagePrice } } + if (!item['O']) { + item['O'] = '非住宅地址' + } params.clear() params["city"] = getCityFormAddress(item['B']) params["address"] = item['B']