From 87d2deb228cffda7dd04a98ac0b5b2f7514285c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 8 May 2021 10:33:05 +0800 Subject: [PATCH 01/23] =?UTF-8?q?=E5=BB=BA=E8=A1=8C=E4=BC=B0=E5=80=BC?= =?UTF-8?q?=E6=96=B9=E6=B3=95=E6=8A=BD=E7=A6=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/weixin/CcbRentController.java | 96 ++++++++++++++++++- .../src/main/webapp/ccbRent/index.ftl | 2 +- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java index 3e97020e7..8b74e93ef 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java @@ -10,9 +10,12 @@ import com.fangjia.dal.mongo.model.RentDealPrice; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.model.District; import com.fangjia.open.compatible.model.JsonModel; +import com.fangjia.open.controllers.PropertyController; import com.fangjia.open.dto.DistrictHouseDtao; import com.fangjia.open.interceptors.AuthenticationBean; import com.fangjia.open.model.BaseModel; +import com.fangjia.open.model.BasicModel; +import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.open.model.weixin.EvaluationModel; import com.fangjia.open.model.weixin.PropertyEvaluationModel; import com.fangjia.open.util.LineChartsUtils; @@ -30,7 +33,6 @@ import org.apache.batik.transcoder.Transcoder; import org.apache.batik.transcoder.TranscoderException; import org.apache.batik.transcoder.TranscoderInput; import org.apache.batik.transcoder.TranscoderOutput; -import org.apache.batik.transcoder.image.JPEGTranscoder; import org.apache.batik.transcoder.image.PNGTranscoder; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.Term; @@ -46,7 +48,6 @@ import org.sothis.web.mvc.annotation.Param; import org.xhtmlrenderer.pdf.ITextFontResolver; import org.xhtmlrenderer.pdf.ITextRenderer; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.*; @@ -87,7 +88,96 @@ public class CcbRentController { return result; } - public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr, + public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + RestfulModelAndView mav = new RestfulModelAndView(); + mav.setViewType("rest"); + if (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getName()) + || StringUtils.isBlank(model.getDistrict()) || model.getSize() <= 0) { + mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市,区域,小区名,面积(size)为必填项")); + return mav; + } + if (model.getCity().endsWith("市")) { + model.setCity(model.getCity().replaceAll("市$", "")); + } + if (StringUtils.isNotBlank(request.getParameter("calDate"))) { + Date calDate = null; + try { + calDate = org.apache.commons.lang.time.DateUtils.parseDate(request.getParameter("calDate"), new String[] { "yyyy-MM-dd" }); + } catch (ParseException e) { + + } + if (calDate == null) { + mav.setModel(BasicModel.buildBadRequestModel("calDate:数据类型不合法")); + return mav; + } else { + model.setCalDate(calDate); + } + } + /* 先到CPDB库中查找小区 */ + District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(), + "district"); + if (district == null) { + mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); + return mav; + } + model.setName(district.getName()); + + + if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) { + mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)")); + return mav; + } + + if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) { + mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)")); + return mav; + } + if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) { + mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层")); + return mav; + } + + model.setRegion(district.getRegion()); + model.setDistrictName(district.getName()); + if (StringUtils.isBlank(model.getPropertyType())) { + if (district.getEstateType2() != null && district.getEstateType2().contains("别墅")) { + model.setPropertyType("1"); + } + } + + if (model.getCalDate() != null) { + if (new Date().before(model.getCalDate())) { + model.setCalDate(DateUtil.getFirstDayOfWeek(new Date())); + } else { + model.setCalDate(DateUtil.getFirstDayOfWeek(model.getCalDate())); + } + } else { + model.setCalDate(DateUtil.getFirstDayOfWeek(new Date())); + } + + Evaluation evaluation = buildEvaluation(model, request); + evaluation.setSource("api_property_evaluate_rent"); + evaluation = evaluationService.evaluateRent(district, evaluation, model.getCalDate()); + + + if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0) { + Map districtPriceMap = new HashMap(); + districtPriceMap.put("city", district.getCity()); + districtPriceMap.put("district", district.getRegion()); + districtPriceMap.put("name", district.getName()); + districtPriceMap.put("address", district.getAddress()); + districtPriceMap.put("totalPrice", evaluation.getTotalPrice()); + districtPriceMap.put("cal_date", model.getCalDate()); + mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap)); + return mav; + } else { + + mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的租金估价").markNoData()); + return mav; + } + } + + public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr, @Param(name = "name") String name, @Param(name = "address") String address, @Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room, @Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) { diff --git a/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl b/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl index 8633e3e89..60f3ac58f 100644 --- a/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl +++ b/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl @@ -260,7 +260,7 @@ $("#evaluateBtn").prop("disabled", true) $.ajax({ // 娉ㄦ剰闇�瑕佸姞.. - url: '../property/evaluateRent', + url: '../ccbRent/evaluateRent', data:$("#evaluateForm").serialize(), dataType:"json", type:"post", From 2ad2c3048029df79ddcb587c042a09eaccc9060c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sat, 8 May 2021 11:03:01 +0800 Subject: [PATCH 02/23] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/AddressController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index c9973367c..e3a690358 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1705,7 +1705,7 @@ public class AddressController { return mav; } - public Object addressSimilarityAction(AddesssMatchModel model) { + public Object similarityAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空")); From c36d79e1f14c783c3585ef6204a5165dd6478c36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Sat, 8 May 2021 13:57:03 +0800 Subject: [PATCH 03/23] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/wenyan/test/Test1.groovy | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy new file mode 100644 index 000000000..fb1b341be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -0,0 +1,26 @@ +package com.fangjia.script.wenyan.test + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject + +ExcelUtils.multiThreadingModify("D:\\数据\\建行地址相似度评级测试.xlsx",{List item-> + def city = item["A"] + def address1 = item["B"] + def address2 = item["C"] + HttpRequest request = new HttpRequest() + def params = new HashMap() + params["city"] = city + params["address_1"] = address1 + params["address_2"] = address2 + params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + JSONObject json = request.getJsonContent("http://192.168.20.86:9180/address/similarity", params, "UTF-8") + JSONObject result = json.result + item["D"] = result?.level + item["E"] = result?.score + item["F"] = result?.formattedAddress_1 + item["G"] = result?.formattedAddress_2 + item["H"] = result?.isSameAddress + println(item) +}) + From 8cb7bd78121ab82e13bb8cb495b65e5fd73d7777 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 8 May 2021 13:46:19 +0800 Subject: [PATCH 04/23] =?UTF-8?q?=E8=AF=84=E4=BC=B0=E6=96=B9=E6=B3=95http?= =?UTF-8?q?=E8=AF=B7=E6=B1=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/weixin/CcbRentController.java | 106 ++++-------------- 1 file changed, 19 insertions(+), 87 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java index 8b74e93ef..c1a70df97 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java @@ -9,13 +9,11 @@ import com.fangjia.dal.mongo.model.Evaluation; import com.fangjia.dal.mongo.model.RentDealPrice; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.model.District; +import com.fangjia.open.APIConstant; import com.fangjia.open.compatible.model.JsonModel; -import com.fangjia.open.controllers.PropertyController; import com.fangjia.open.dto.DistrictHouseDtao; import com.fangjia.open.interceptors.AuthenticationBean; import com.fangjia.open.model.BaseModel; -import com.fangjia.open.model.BasicModel; -import com.fangjia.open.model.RestfulModelAndView; import com.fangjia.open.model.weixin.EvaluationModel; import com.fangjia.open.model.weixin.PropertyEvaluationModel; import com.fangjia.open.util.LineChartsUtils; @@ -51,6 +49,7 @@ import org.xhtmlrenderer.pdf.ITextRenderer; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.*; +import java.lang.reflect.InvocationTargetException; import java.text.ParseException; import java.util.*; @@ -89,92 +88,25 @@ public class CcbRentController { } public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { - RestfulModelAndView mav = new RestfulModelAndView(); - mav.setViewType("rest"); - if (StringUtils.isBlank(model.getCity()) || StringUtils.isBlank(model.getName()) - || StringUtils.isBlank(model.getDistrict()) || model.getSize() <= 0) { - mav.setModel(BasicModel.buildBadRequestModel("参数非法:城市,区域,小区名,面积(size)为必填项")); - return mav; - } - if (model.getCity().endsWith("市")) { - model.setCity(model.getCity().replaceAll("市$", "")); - } - if (StringUtils.isNotBlank(request.getParameter("calDate"))) { - Date calDate = null; - try { - calDate = org.apache.commons.lang.time.DateUtils.parseDate(request.getParameter("calDate"), new String[] { "yyyy-MM-dd" }); - } catch (ParseException e) { + String result = null; + HttpRequest httpRequest = new HttpRequest(); + try { + Map headers = new HashMap(); - } - if (calDate == null) { - mav.setModel(BasicModel.buildBadRequestModel("calDate:数据类型不合法")); - return mav; - } else { - model.setCalDate(calDate); - } - } - /* 先到CPDB库中查找小区 */ - District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(), - "district"); - if (district == null) { - mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); - return mav; - } - model.setName(district.getName()); - - - if (model.getMaxFloor() != null && (model.getMaxFloor() <= 0 || model.getMaxFloor() > 59)) { - mav.setModel(BasicModel.buildBadRequestModel("maxFloor取值非法,正确范围[1,60)")); - return mav; - } - - if (model.getFloor() != null && (model.getFloor() <= 0 || model.getFloor() > 59)) { - mav.setModel(BasicModel.buildBadRequestModel("floor取值非法,正确范围[1,60)")); - return mav; - } - if (model.getMaxFloor() != null && model.getFloor() != null && model.getFloor() > model.getMaxFloor()) { - mav.setModel(BasicModel.buildBadRequestModel("所在楼层不能大于总楼层")); - return mav; - } - - model.setRegion(district.getRegion()); - model.setDistrictName(district.getName()); - if (StringUtils.isBlank(model.getPropertyType())) { - if (district.getEstateType2() != null && district.getEstateType2().contains("别墅")) { - model.setPropertyType("1"); - } - } - - if (model.getCalDate() != null) { - if (new Date().before(model.getCalDate())) { - model.setCalDate(DateUtil.getFirstDayOfWeek(new Date())); - } else { - model.setCalDate(DateUtil.getFirstDayOfWeek(model.getCalDate())); - } - } else { - model.setCalDate(DateUtil.getFirstDayOfWeek(new Date())); - } - - Evaluation evaluation = buildEvaluation(model, request); - evaluation.setSource("api_property_evaluate_rent"); - evaluation = evaluationService.evaluateRent(district, evaluation, model.getCalDate()); - - - if (evaluation.getTotalPrice() != null && evaluation.getTotalPrice() != 0) { - Map districtPriceMap = new HashMap(); - districtPriceMap.put("city", district.getCity()); - districtPriceMap.put("district", district.getRegion()); - districtPriceMap.put("name", district.getName()); - districtPriceMap.put("address", district.getAddress()); - districtPriceMap.put("totalPrice", evaluation.getTotalPrice()); - districtPriceMap.put("cal_date", model.getCalDate()); - mav.setModel(BasicModel.buildOKModel().setResult(districtPriceMap)); - return mav; - } else { - - mav.setModel(BasicModel.buildOKModel().setMsg("暂时不支持该小区的租金估价").markNoData()); - return mav; + final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath(); + String url = path + "/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN + "&__refresh_cache=1"; + Map params = new HashMap(); + params = org.apache.commons.beanutils.BeanUtils.describe(model); + //params.put("request", JsonUtils.httpParameterToJSONObject(request)); + result = httpRequest.post(url, params, headers,"UTF-8"); + } catch (HttpRequestException e) { + e.printStackTrace(); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); } + return result; } public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr, From 3fda9d7f8a810c4336b6f8bc6d60285310668297 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 8 May 2021 14:07:16 +0800 Subject: [PATCH 05/23] =?UTF-8?q?=E5=8E=BB=E9=87=8D=E7=BC=93=E5=AD=98?= =?UTF-8?q?=E5=88=B7=E6=96=B0=EF=BC=8C=E8=BF=9E=E6=8E=A5=E5=89=8D=E7=BC=80?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/weixin/CcbRentController.java | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java index c1a70df97..3a17ffc8b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java @@ -92,17 +92,14 @@ public class CcbRentController { HttpRequest httpRequest = new HttpRequest(); try { Map headers = new HashMap(); - - final String path = "http://" + request.getServerName() + ":" + request.getServerPort() + request.getContextPath(); - String url = path + "/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN + "&__refresh_cache=1"; - Map params = new HashMap(); - params = org.apache.commons.beanutils.BeanUtils.describe(model); - //params.put("request", JsonUtils.httpParameterToJSONObject(request)); + // &__refresh_cache=1 刷新缓存 + String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN; + Map params = org.apache.commons.beanutils.BeanUtils.describe(model); result = httpRequest.post(url, params, headers,"UTF-8"); - } catch (HttpRequestException e) { - e.printStackTrace(); } catch (NoSuchMethodException e) { e.printStackTrace(); + } catch (HttpRequestException e) { + e.printStackTrace(); } catch (InvocationTargetException e) { e.printStackTrace(); } From f69c8bd02801029d5d5ddb502bcc34836a931b3a Mon Sep 17 00:00:00 2001 From: xuhe Date: Sun, 9 May 2021 12:34:36 +0800 Subject: [PATCH 06/23] =?UTF-8?q?=E7=9B=B8=E4=BC=BC=E5=BA=A6=E8=AF=84?= =?UTF-8?q?=E7=BA=A7=E4=BB=A3=E7=A0=81=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../service/web/AddressServiceImpl.java | 62 +++++++++---------- 1 file changed, 30 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 221fecc5b..8b8a1f1fc 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5,18 +5,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.net.URLEncoder; import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -5138,30 +5127,39 @@ public class AddressServiceImpl implements AddressService { return null; } /** - * 获取两地址的相似度评级(最小颗粒度) + * 获取两地址的相似度评级(相同数据的最小颗粒度) * @return */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ - if(StringUtils.isNotBlank(getFromJson(json1,"roomNumber")) || StringUtils.isNotBlank(getFromJson(json2,"roomNumber"))){ - return "roomNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"unitNumber")) || StringUtils.isNotBlank(getFromJson(json2,"unitNumber"))){ - return "unitNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"buildingNumber")) || StringUtils.isNotBlank(getFromJson(json2,"buildingNumber"))){ - return "buildingNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"name")) || StringUtils.isNotBlank(getFromJson(json2,"name"))){ - return "name"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"street")) || StringUtils.isNotBlank(getFromJson(json2,"street"))){ - return "street"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"township")) || StringUtils.isNotBlank(getFromJson(json2,"township"))){ - return "township"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"district")) || StringUtils.isNotBlank(getFromJson(json2,"district"))){ - return "district"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"city")) || StringUtils.isNotBlank(getFromJson(json2,"city"))){ - return "city"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"province")) || StringUtils.isNotBlank(getFromJson(json2,"province"))){ - return "province"; + String level = null; + LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street" + , "name", "buildingNumber", "unitNumber", "roomNumber")); + for(String key:list){ + if(continueJudgeValue(json1, json2, key) == 3){ + return level; + }else{ + level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level); + } } - return null; + return level; + } + + /** + * 判断level时是否继续往下判断 + * @param json1 + * @param json2 + * @param key + * @return 1 继续判断并且level有值 2继续判断level没值 3 停止 + */ + private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ + if(getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } + return 2; } /** * 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true) From 2469d03c29b694a9e5049c072bc5f683204df637 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Sun, 9 May 2021 16:34:56 +0800 Subject: [PATCH 07/23] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Dockers=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/web/AddressServiceImpl.java | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 8b8a1f1fc..1e04da6c0 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -2348,11 +2348,10 @@ public class AddressServiceImpl implements AddressService { /** * 基于高德开放接口的地址真实性检查 - * * @param city * @param address * @param checkBo - * @param time + * @param addressList */ private void checkAmapNew(String city, String address, AddressCheckBo checkBo, Set addressList) { try { @@ -3571,7 +3570,7 @@ public class AddressServiceImpl implements AddressService { * * @param city * @param address - * @param checkBo + * @param keywordsFormat */ private AddressCheckTruth checkTruthSplit(String city, String address, Map keywordsFormat) { // 1.拆分 @@ -4580,8 +4579,7 @@ public class AddressServiceImpl implements AddressService { /** * 地址评级具体逻辑 - * - * @param checkBo + * @param checkTr * @return */ private AddressCheckTruth gradeAddressTruth(AddressCheckTruth checkTr) { @@ -4604,8 +4602,8 @@ public class AddressServiceImpl implements AddressService { /** * 小区评分 - * - * @param checkBo + * @param check + * @param isConsume * @return */ private boolean gradeDistrictTruth(AddressCheckTruth check,boolean isConsume) { @@ -5132,7 +5130,7 @@ public class AddressServiceImpl implements AddressService { */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ String level = null; - LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street" + LinkedList list = new LinkedList(Arrays.asList("province", "city", "district", "township", "street" , "name", "buildingNumber", "unitNumber", "roomNumber")); for(String key:list){ if(continueJudgeValue(json1, json2, key) == 3){ From 4d4602d7ad9a51d86c34e719b84674a28a6c70d9 Mon Sep 17 00:00:00 2001 From: xuhe Date: Sun, 9 May 2021 17:39:25 +0800 Subject: [PATCH 08/23] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E6=A0=87=E5=87=86?= =?UTF-8?q?=E5=8C=96=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 1 + .../address/AddressParserServiceImpl.java | 176 ++++++++++++++---- .../fangjia/service/address/mode/Address.java | 10 + .../service/address/mode/MatchParams.java | 8 + .../processor/StageAndPartitionProcessor.java | 1 + .../service/api/AliyunServiceImpl.java | 72 +++++-- .../service/web/AddressServiceImpl.java | 2 +- 7 files changed, 222 insertions(+), 48 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index e3a690358..96bbe097f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1769,6 +1769,7 @@ public class AddressController { // matchParams.setUseMap(true); matchParams.setUseAliyunApi(true); +// matchParams.setUseCrf(true); matchParams.setContainsLocation(model.isContainsLocation()); Building building = addressParserService.standard(matchParams); Address address = building.getSearchAddress(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 8bc805491..9eb8f4114 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,14 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.CNNMFilter; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.GeoUtils; -import com.fangjia.core.util.StringUtil; +import com.fangjia.core.util.*; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; +import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -25,6 +23,7 @@ import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.bson.types.ObjectId; @@ -134,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService { formatAddress(building); } return building; + /*JSONObject result = getDataFromCRF("test"); + return new Building();*/ } @Override @@ -165,6 +166,21 @@ public class AddressParserServiceImpl implements AddressParserService { return building; } Building matchedBuilding; + //调用CRF算法切分数据 + /*JSONObject jsonObj = null; + if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + jsonObj = getDataFromCRF(addr.getOrgAddress()); + if(jsonObj != null){ + String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") + + (jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") + + (jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : ""); + if(StringUtils.isNotBlank(poi)){ + if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){ + addr.setKeywords(poi); + } + } + } + }*/ //keywords使用aliyun接口返回的poi AliyunBo aliyunBo = null; if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ @@ -176,6 +192,9 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setKeywords(poi); } } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(aliyunBo.getCity()); + } if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){ String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length())); if(StringUtils.isNotBlank(str)){ @@ -187,26 +206,80 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - if(StringUtils.isNotBlank(aliyunBo.getCity())){ - addr.setSuggestCity(aliyunBo.getCity()); - } - } - } - AddressMatchMetaData meta = null; - if(StringUtils.isBlank(addr.getCity())){ - addr.addKeyword(addr.getOrgAddress()); - meta = getMapMeta(addr); - if(meta != null){ - addr.setProvince(meta.getMapProvince()); - addr.setCity(meta.getMapCity()); - addr.setDistrict(meta.getDistrict()); } } + addr.addKeyword(addr.getOrgAddress()); if(params.isUseMap()){ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP); }else{ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB); } + + /*if(jsonObj != null){ + //乡镇优化 + String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : ""; + if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){ + String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township); + if(StringUtils.isNotBlank(town)){ + addr.setTownship(town); + } + } + //道路优化(两边均去掉乡镇) + String street = addr.getStreet(); + String streetNumber = addr.getStreetNumber(); + String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : ""; + String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : ""; + //去掉路名中含有乡镇名的 + if(StringUtils.isNotBlank(addr.getTownship())){ + street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street; + streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf; + } + + //crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理 + if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){ + //Java解析出路名为空 + if(StringUtils.isBlank(street)){ + street = streetCrf; + if(StringUtils.isNotBlank(streetNumberCrf)){ + streetNumber = streetNumberCrf; + } + }else{ + //CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则 + if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){ + street = streetCrf; + } + } + addr.setStreet(street); + addr.setStreetNumber(streetNumber); + } + //poi处理 + //没有匹配上库中小区时 + if(StringUtils.isBlank(matchedBuilding.getId())){ + String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") + + (jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") + + (jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : ""); + //去掉路名中含有乡镇名和路名 getParsedPropertyName + if(StringUtils.isNotBlank(addr.getTownship())){ + poi = poi.replace(addr.getTownship(),""); + } + if(StringUtils.isNotBlank(addr.getStreet())){ + poi = poi.replace(addr.getStreet(),""); + } + addr.setParsedPropertyName(poi); + } + //楼栋、单元、室号 + String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + addr.setOrgBuildingNumber(houseNo); + addr.setBuildingNumber(houseNo); + addr.setOrgUnitNumber(cellNo); + addr.setUnitNumber(cellNo); + addr.setFloor(floorNo); + addr.setOrgRoomNumber(roomNo); + addr.setRoomNumber(roomNo); + }*/ matchedBuilding.setProvince(addr.getProvince()); matchedBuilding.setSearchAddress(addr); matchedBuilding.setSuggestName(getSuggestName(matchedBuilding)); @@ -215,7 +288,8 @@ public class AddressParserServiceImpl implements AddressParserService { completBuildingField(matchedBuilding, aliyunBo, params); } Address tempAddr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(tempAddr.getStreet()) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + AddressMatchMetaData meta = getMapMeta(addr); if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){ matchedBuilding.setAddress(meta.getAddress()); } @@ -255,6 +329,25 @@ public class AddressParserServiceImpl implements AddressParserService { } return null; } + + /** + * TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用 + * @param address + * @return + */ + /*private JSONObject getDataFromCRF(String address){ + HttpRequest request = new HttpRequest(); + Map params = new HashMap<>(); + params.put("address",address); + try { + JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8"); + System.out.println(resp); + return resp; + }catch(Exception e){ + + } + return null; + }*/ /** * 信息不全时使用阿里接口补全数据 * @param matchedBuilding @@ -271,28 +364,39 @@ public class AddressParserServiceImpl implements AddressParserService { } } Address addr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(addr.getProvince())){ + if(StringUtils.isNotBlank(aliyunBo.getProv())){ + addr.setProvince(formatProvince(aliyunBo.getProv())); + } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } + if(StringUtils.isNotBlank(aliyunBo.getDistrict())){ + addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); + } + addr.setTownship(aliyunBo.getTown()); + + /*if(StringUtils.isBlank(addr.getProvince())){ addr.setProvince(formatProvince(aliyunBo.getProv())); } if(StringUtils.isBlank(addr.getCity())){ addr.setCity(RegionData.formatCity(aliyunBo.getCity())); } - - if(StringUtils.isBlank(addr.getDevelopmentZone())){ - addr.setDevelopmentZone(aliyunBo.getDevzone()); - } if(StringUtils.isBlank(addr.getDistrict())){ addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); } if(StringUtils.isBlank(addr.getTownship())){ addr.setTownship(aliyunBo.getTown()); + }*/ + if(StringUtils.isBlank(addr.getDevelopmentZone())){ + addr.setDevelopmentZone(aliyunBo.getDevzone()); } + /*addr.setOrgBuildingNumber(aliyunBo.getHouseno()); addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); addr.setUnitNumber(aliyunBo.getCellno());*/ if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ - addr.setRoomNumber(aliyunBo.getRoomno()); + addr.setOrgRoomNumber(aliyunBo.getRoomno()); addr.setRoomNumber(aliyunBo.getRoomno()); }else{ addr.setRoomNumber(null); @@ -1037,20 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService { if(StringUtils.isNotBlank(address.getTownship())){ formattedAddressBuilder.append(address.getTownship()); } + if(StringUtils.isNotBlank(building.getAddress())){ + Address orgAddr = new Address(); + orgAddr.setOrgAddress(building.getAddress()); + Address addr = AddressProcessors.process(orgAddr); + if(StringUtils.isBlank(address.getStreet())){ + address.setStreet(addr.getStreet()); + } + if(StringUtils.isBlank(address.getStreetNumber())){ + address.setStreetNumber(addr.getStreetNumber()); + } + } if(StringUtils.isNotBlank(address.getStreet())){ formattedAddressBuilder.append(address.getStreet()); } if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){ formattedAddressBuilder.append(address.getStreetNumber()); } - if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ - Address orgAddr = new Address(); - orgAddr.setOrgAddress(building.getAddress()); - Address addr = AddressProcessors.process(orgAddr); - building.getSearchAddress().setStreet(addr.getStreet()); - building.getSearchAddress().setStreetNumber(addr.getStreetNumber()); - formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber())); - } + if(StringUtils.isNotBlank(building.getSuggestName())){ formattedAddressBuilder.append(building.getSuggestName()); } @@ -1077,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ - formattedAddressBuilder.append(address.getRoomNumber()); -// formattedAddressBuilder.append(address.getRoomNumber() + "室"); +// formattedAddressBuilder.append(address.getRoomNumber()); + formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 58022cf03..1c8f2d004 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -77,6 +77,16 @@ public class Address { private String debugName; private AliyunBo aliyunBo; + //使用CRF切分的地址 + private String crfAddress; + + public String getCrfAddress() { + return crfAddress; + } + + public void setCrfAddress(String crfAddress) { + this.crfAddress = crfAddress; + } public AddressParserGoal getAddressParserGoal() { return addressParserGoal; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java index 498b12052..e53c9701f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java @@ -25,7 +25,15 @@ public class MatchParams { private boolean useMap; /**是否使用阿里云接口*/ private boolean useAliyunApi; + //是否使用CRF算法 + private boolean useCrf; + public boolean isUseCrf() { + return useCrf; + } + public void setUseCrf(boolean useCrf) { + this.useCrf = useCrf; + } public boolean isContainsLocation() { return containsLocation; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java index 93c69b4c4..ab5c929b3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java @@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor { @Override public RepeatStatus process(Address address) { String orgAddr = address.getEffectiveAddress(); + address.setCrfAddress(address.getAddress()); MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress()); if (matchResult != null) { String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index b6b22f5a1..bcbdba418 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -1,5 +1,7 @@ package com.fangjia.service.api; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse; import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest; import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse; import com.fangjia.core.util.JsonUtils; @@ -13,6 +15,9 @@ import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; +import java.util.ArrayList; +import java.util.List; + public class AliyunServiceImpl implements AliyunService { protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); @@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService { if(StringUtils.isNotBlank(city)){ request.setDefaultCity(city); } + AliyunBo aliyunBo = new AliyunBo(); try{ StructureAddressResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); String value = (String) responseJson.get("structure"); - AliyunBo aliyunBo = new AliyunBo(); for (String key:value.split("\t")){ String[] arry = key.split("="); wrapField(arry[0],arry[1], aliyunBo); } - return aliyunBo; }catch(Exception e){ - LOGGER.error("", e); - return null; + LOGGER.error("调用阿里标准化接口失败:", e); } + try{ + GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest(); + divisionRequest.setServiceCode("addrp"); + divisionRequest.setAppKey(APPKEY); + divisionRequest.setText(address); + GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest); + JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData()); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList<>(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + aliyunBo.setProv(array[0]); + aliyunBo.setCity(array[1]); + aliyunBo.setDistrict(array[2]); + aliyunBo.setTown(array[3]); + }catch(Exception e){ + LOGGER.error("调用阿里行政区划识别接口失败", e); + } + return aliyunBo; } private void wrapField(String field,String value, AliyunBo aliyunBo){ @@ -86,21 +115,38 @@ public class AliyunServiceImpl implements AliyunService { ACCESSKEYID, SECRET); IAcsClient client = new DefaultAcsClient(profile); - StructureAddressRequest request = new StructureAddressRequest(); +// StructureAddressRequest request = new StructureAddressRequest(); + GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest(); request.setServiceCode("addrp"); request.setAppKey(APPKEY); - request.setText("杭州市文一西路969号"); - request.setDefaultProvince("浙江省"); - request.setDefaultCity("杭州市"); - request.setDefaultDistrict("余杭区"); - StructureAddressResponse response = client.getAcsResponse(request); + request.setText("北郭乡东草亭村一号路2号"); +// request.setDefaultProvince("浙江省"); +// request.setDefaultCity("杭州市"); +// request.setDefaultDistrict("余杭区"); +// StructureAddressResponse response = client.getAcsResponse(request); + GetAddressDivisionCodeResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); - String value = (String) responseJson.get("structure"); - JSONObject json = new JSONObject(); + System.out.println(responseJson); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]); + /*JSONObject json = new JSONObject(); for (String key:value.split("\t")){ String[] arry = key.split("="); json.put(arry[0],arry[1]); } - System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class)); + System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/ } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 8b8a1f1fc..67bc70049 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5132,7 +5132,7 @@ public class AddressServiceImpl implements AddressService { */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ String level = null; - LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street" + LinkedList list = new LinkedList<>(Arrays.asList("province", "city", "district", "township", "street", "streetNumber" , "name", "buildingNumber", "unitNumber", "roomNumber")); for(String key:list){ if(continueJudgeValue(json1, json2, key) == 3){ From 3939ca595f46ebac45f7ea21e31b19116cf59125 Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Sun, 9 May 2021 19:54:49 +0800 Subject: [PATCH 09/23] =?UTF-8?q?=E6=B7=BB=E5=8A=A0Dockers=E6=96=87?= =?UTF-8?q?=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/groovy/com/fangjia/script/wenyan/test/Test1.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index fb1b341be..cafe803de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -4,7 +4,7 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject -ExcelUtils.multiThreadingModify("D:\\数据\\建行地址相似度评级测试.xlsx",{List item-> +ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] def address2 = item["C"] @@ -14,7 +14,7 @@ ExcelUtils.multiThreadingModify("D:\\ params["address_1"] = address1 params["address_2"] = address2 params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" - JSONObject json = request.getJsonContent("http://192.168.20.86:9180/address/similarity", params, "UTF-8") + JSONObject json = request.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") JSONObject result = json.result item["D"] = result?.level item["E"] = result?.score From d2aab5988970f0b120eed24bdd5d6fa1ddcf3cab Mon Sep 17 00:00:00 2001 From: wangwenyan Date: Sun, 9 May 2021 19:56:15 +0800 Subject: [PATCH 10/23] fix bug --- .../main/java/com/fangjia/service/api/AliyunServiceImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index bcbdba418..cf02bf903 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -62,7 +62,7 @@ public class AliyunServiceImpl implements AliyunService { String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); String[] array = value.replace("\t", ";").split(";"); if(array.length > 4){ - List tmp = new ArrayList<>(); + List tmp = new ArrayList(); for(String str:array){ if(str!=null && str.length()!=0){ tmp.add(str); From 0b0cd747314a9ed667b439b0cd89c1c369874fad Mon Sep 17 00:00:00 2001 From: xuhe Date: Sun, 9 May 2021 22:37:07 +0800 Subject: [PATCH 11/23] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=9B=B8=E4=BC=BC?= =?UTF-8?q?=E5=BA=A6=E8=AF=84=E7=BA=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/AddressParserServiceImpl.java | 6 ++--- .../service/api/AliyunServiceImpl.java | 22 ++++++++++++++++--- .../service/web/AddressServiceImpl.java | 17 +++++++++----- 3 files changed, 34 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 9eb8f4114..2caa0b7a3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -193,7 +193,7 @@ public class AddressParserServiceImpl implements AddressParserService { } } if(StringUtils.isNotBlank(aliyunBo.getCity())){ - addr.setCity(aliyunBo.getCity()); + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); } if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){ String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length())); @@ -1185,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ -// formattedAddressBuilder.append(address.getRoomNumber()); - formattedAddressBuilder.append(address.getRoomNumber() + "室"); + formattedAddressBuilder.append(address.getRoomNumber()); +// formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index cf02bf903..91d309485 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -110,7 +110,23 @@ public class AliyunServiceImpl implements AliyunService { } } public static void main(String[] args) throws ClientException { - DefaultProfile profile = DefaultProfile.getProfile( + String value = ""; + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]); + + /*DefaultProfile profile = DefaultProfile.getProfile( REGIONID, ACCESSKEYID, SECRET); @@ -119,7 +135,7 @@ public class AliyunServiceImpl implements AliyunService { GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest(); request.setServiceCode("addrp"); request.setAppKey(APPKEY); - request.setText("北郭乡东草亭村一号路2号"); + request.setText("天青路260号"); // request.setDefaultProvince("浙江省"); // request.setDefaultCity("杭州市"); // request.setDefaultDistrict("余杭区"); @@ -141,7 +157,7 @@ public class AliyunServiceImpl implements AliyunService { System.out.println(array[0]); System.out.println(array[1]); System.out.println(array[2]); - System.out.println(array[3]); + System.out.println(array[3]);*/ /*JSONObject json = new JSONObject(); for (String key:value.split("\t")){ String[] arry = key.split("="); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index c236acbb4..de2a772b2 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5069,7 +5069,7 @@ public class AddressServiceImpl implements AddressService { double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin); String level = getAddressSimilarityLevel(json1, json2); resultMap.put("isSameAddress", sameAddress); - resultMap.put("score", similar); +// resultMap.put("score", similar); resultMap.put("level", level); resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress")); resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress")); @@ -5153,10 +5153,14 @@ public class AddressServiceImpl implements AddressService { */ private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ - if(getFromJson(json1,key).equals(getFromJson(json2,key))){ - return 1; + if("name".equals(key)){ + return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3; }else{ - return 3; + if(getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } } } return 2; @@ -5208,6 +5212,10 @@ public class AddressServiceImpl implements AddressService { if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){ return false; } + return isSameName(name1, name2); + } + + private boolean isSameName(String name1, String name2){ String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1); String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2); //拼音或文本数据包含并且 @@ -5220,7 +5228,6 @@ public class AddressServiceImpl implements AddressService { } return false; } - /** * 设置区域消费指数 * @param checkTr From 4afd40a650695a8e49ad4d964fc6c46a721a4397 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Mon, 10 May 2021 18:40:55 +0800 Subject: [PATCH 12/23] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8?= =?UTF-8?q?=E7=89=88=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/wenyan/test/Test1.groovy | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index cafe803de..401d647e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -1,26 +1,46 @@ package com.fangjia.script.wenyan.test +import com.aliyuncs.DefaultAcsClient +import com.aliyuncs.IAcsClient +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse +import com.aliyuncs.profile.DefaultProfile import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject -ExcelUtils.multiThreadingModify("C:\\Users\\wenyan\\Downloads\\建行地址相似度评级测试.xlsx",{List item-> +ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] def address2 = item["C"] - HttpRequest request = new HttpRequest() + HttpRequest http = new HttpRequest() def params = new HashMap() params["city"] = city params["address_1"] = address1 params["address_2"] = address2 params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" - JSONObject json = request.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") + JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") JSONObject result = json.result item["D"] = result?.level - item["E"] = result?.score - item["F"] = result?.formattedAddress_1 - item["G"] = result?.formattedAddress_2 - item["H"] = result?.isSameAddress - println(item) + item["E"] = result?.formattedAddress_1 + item["F"] = result?.formattedAddress_2 + item["G"] = result?.isSameAddress + DefaultProfile profile = DefaultProfile.getProfile( + "cn-hangzhou", + "LTAIdaj4eSO79gzz", + "xw6EsvqydZ9HC6zAvIB3ruQqVHynxf"); + IAcsClient client = new DefaultAcsClient(profile); + GetAddressSimilarityRequest request = new GetAddressSimilarityRequest(); + request.setServiceCode("addrp") + request.setAppKey("Xcgtyhupey0h") + request.setText("${address1};${address2}") + GetAddressSimilarityResponse response = client.getAcsResponse(request) + String data = response.getData() + json = JSONObject.fromObject(data) + item["H"] = json["sim_level"]["level"] + item["I"] = json["sim_level"]["address_a"] + item["J"] = json["sim_level"]["address_b"] + item["K"] = json["sim_level"]["is_same"] + println(json) }) From 07c166e84312ba51c8331372079fcee535998f0a Mon Sep 17 00:00:00 2001 From: xuhe Date: Mon, 10 May 2021 23:13:21 +0800 Subject: [PATCH 13/23] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E7=9B=B8=E4=BC=BC?= =?UTF-8?q?=E5=BA=A6=E8=AF=84=E7=BA=A7=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/service/address/AddressParserServiceImpl.java | 8 ++++---- .../java/com/fangjia/service/web/AddressServiceImpl.java | 8 ++++++++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 2caa0b7a3..1512e2e7e 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -395,13 +395,13 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); addr.setUnitNumber(aliyunBo.getCellno());*/ - if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ + /*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ addr.setOrgRoomNumber(aliyunBo.getRoomno()); addr.setRoomNumber(aliyunBo.getRoomno()); }else{ addr.setRoomNumber(null); addr.setRoomNumber(null); - } + }*/ /*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", ""); String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", ""); @@ -1185,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ - formattedAddressBuilder.append(address.getRoomNumber()); -// formattedAddressBuilder.append(address.getRoomNumber() + "室"); +// formattedAddressBuilder.append(address.getRoomNumber()); + formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index de2a772b2..21ab2139d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5152,6 +5152,14 @@ public class AddressServiceImpl implements AddressService { * @return 1 继续判断并且level有值 2继续判断level没值 3 停止 */ private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ + if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){ + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key)) + && getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ if("name".equals(key)){ return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3; From 947853db4d56200b5965317178530151691499f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Wed, 12 May 2021 13:17:46 +0800 Subject: [PATCH 14/23] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A1=97=E9=81=93?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/dal/seaweed/model/District.java | 22 +++--- .../com/fangjia/script/wenyan/Test4.groovy | 70 ++++++++++++------- .../address/AddressParserServiceImpl.java | 2 +- .../service/address/mode/Building.java | 8 +-- 4 files changed, 56 insertions(+), 46 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index 401132cb0..afba53e15 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -122,10 +122,8 @@ public class District implements SeaweedEntity { private String houseStructure; //房屋结构 private String source; private String propertyLife; -// -// //乡镇街道 -// private String township; -// + //乡镇街道 + private String township; private String pinyin; @@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity { this.mappingAlias = mappingAlias; } -// @Column(name = "township") -// public String getTownship() { -// return township; -// } -// -// public void setTownship(String township) { -// this.township = township; -// } + @Column(name = "township") + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index bef30856c..95d41237b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -6,34 +6,50 @@ import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +def count = 0 +List> data = Collections.synchronizedList([]) +data << ["ID","城市","区域","小区名","经度","纬度","街道"] - -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\数据.xlsx",{List item-> -// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){ -// item["D"] = "存在" -// }else{ -// item["D"] = "不存在" -// } -//}) - - -DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com") -ProvinceData.getAllCity("福建").each {def city-> - int count = 0 - def s = new TreeMap() - DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat-> - count ++ - int credit = (stat.credit * 100)/10 - s[credit] = s.get(credit, 0) + 1 - } - - def data = [] - data << city - data << count - for(int i=0; i <=10; i++){ - data << s.get(i, 0) - } - println(data.join(",")) +CommonUtils.findAllCity().each {def city-> + DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> + if(StringUtils.isBlank(item.township) && item.lng){ + count++ + def lng = item.lng/10000000000d + def lat = item.lat/10000000000d + def location = "${lng},${lat}" + JSONObject resp = GaoDeApiUtils.geocode(location) + def row = [] + row << item._id.toString() + row << item.city + row << item.region + row << item.name + row << lng + row << lat + BasicDBObject dbo = new BasicDBObject() + dbo.id = item._id.toString() + dbo.city = item.city + dbo.region = item.region + dbo.name = item.name + dbo.lng = lng + dbo.lat = lat + if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ + row << resp.regeocode.addressComponent.township + dbo.township = resp.regeocode.addressComponent.township + } + println(row) + data << row + SEAWEED_TOWNSHIP.insert(dbo) + } + }) } +ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") + + + + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 1512e2e7e..46545fc04 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -46,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService { private LoudongService loudongService; private SeaweedDao seaweedDao; private AliyunService aliyunService; - private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"}; + private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"}; private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java index a217fa8fd..251470ffc 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java @@ -64,9 +64,7 @@ public class Building { this.alias = d.getAlias(); this.addrAlias = d.getAddrAlias(); this.propertyType = d.getEstateType2(); - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); if(d.getLat() != null && d.getLng() != null){ this.lat = d.getLat()/10000000000d; this.lng = d.getLng()/10000000000d; @@ -90,9 +88,7 @@ public class Building { this.lat = d.getLat2()/10000000000d; this.lng = d.getLng2()/10000000000d; } - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); } public Building() { From 1f0a3177199d152e19f24d546a4529d2f1b78b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 16:46:56 +0800 Subject: [PATCH 15/23] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E8=AD=A6=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/Watcher.groovy | 1 + .../temp/shenjian/SeawaterMonitor.groovy | 148 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy index 85088ddd8..004d6bd4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy @@ -32,6 +32,7 @@ public enum Watcher { BaYi('15300424133','bayi@fangjia.com'), WeiLei('18101751850','weilei@fangjia.com'), JiangYaHui('13684913780','jiangyahui@fangjia.com'), + ShenJian('13764369683', 'shenjian@fangjia.com') private final String mobile; private final String email; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy new file mode 100644 index 000000000..a2d5c158d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -0,0 +1,148 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import org.apache.commons.lang.StringUtils +import org.apache.commons.lang.time.DateUtils + +/** + * 抓取总量监控 + * + * @author shenjian + * @since 2021/5/12 + */ +//@BatchOwner(Watcher.ShenJian) +//class SeawaterMonitor extends BaseBatch { +class SeawaterMonitor { + + static final Long expectNumber = 400000; + // 运维组 运维警报 + static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + // 重点城市 + static final city = ["北京','唐山','天津','秦皇岛','石家庄','包头','太原','丹东','呼和浩特','锦州'," + + "'沈阳','吉林','大连','牡丹江','长春','无锡','哈尔滨','扬州','上海','徐州'," + + "'南京','温州','杭州','金华','宁波','蚌埠','合肥','安庆','福州','泉州'," + + "'厦门','九江','南昌','赣州','济南','烟台','青岛','济宁','郑州','洛阳'," + + "'武汉','平顶山','长沙','宜昌','广州','襄阳','深圳','岳阳','南宁','常德'," + + "'海口','惠州','重庆','湛江','成都','韶关','贵阳','桂林','昆明','北海'," + + "'西安','三亚','兰州','泸州','西宁','南充','银川','遵义','乌鲁木齐','大理'," + + "'东莞','佛山','苏州','珠海','中山','台州','绍兴','南通','常州','嘉兴'," + + "'淄博','株洲','镇江','肇庆','漳州','盐城','芜湖','泰州','汕头','清远'," + + "'马鞍山','柳州','连云港','廊坊','淮安','湖州','沧州','保定','张家口','日照'," + + "'拉萨','大同','文昌','陵水','保亭"]; + // 重点网站 + static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] + + static void main(String[] args) { + def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") + def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") + def startDate = yesterday + "T00:00:00" + def endDate = yesterday + "T23:59:59" + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } + + /** + * 监控前一日抓取量 + * @return + */ + static def monitorYesterdayCrawlNumber(def startDate, def endDate) { + def total + String sql = "select count(*) as total from latest_seawater where m_date between '${startDate}' and '${endDate}' " + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + if (total < expectNumber) { + return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber + } + return "" + } + + /** + * 统计重点城市近半月抓取量 + * @return + */ + static boolean countCityCrawlNumber(def startDate, def endDate) { + def result = [] + result << ['城市', '抓取量'] + // 统计近半个月各重点城市抓取量 + def cityFormatted = city.join("','") + String sql = "select * from (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + ElasticUtils.executeSqlQuery(sql,{data-> + result << [data.city, data.total] + }) + ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量") + } + + /** + * 统计前一日重点城市抓取量,昨天为零则警报 + * @return + */ + static def monitorCityCrawlNumber(def startDate, def endDate) { + def msg = "【重点城市抓取异常】" + def unNormal = [] + def cityFormatted = city.join("','") + String sql = "select * from " + + " (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) where total=0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + unNormal.add(data.city) + }) + msg += unNormal.join("、") + " 抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + + /** + * 统计重点网站抓取情况,近一周为0则警报 + */ + static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) { + def msg = "【重点网站抓取异常】" + def set = new HashSet() + def siteFormatted = site.join("','") + String sql = "select * from " + + " (select count(*) as total, root from latest_seawater where m_date between '${lastWeekDate}' and '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + set.add(data.root) + }) + def unNormal = [] + for (def info : site) { + if (!set.contains(info)) { + unNormal.add(info) + } + } + msg += unNormal.join("、") + " 近一周抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + +// @Override +// void process(Object args) { +// +// } +} From c0484532090cdce60d8bc20a7f93ecfaf35e9ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 18:43:20 +0800 Subject: [PATCH 16/23] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E8=AD=A6=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index a2d5c158d..60e2419de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil @@ -12,9 +15,8 @@ import org.apache.commons.lang.time.DateUtils * @author shenjian * @since 2021/5/12 */ -//@BatchOwner(Watcher.ShenJian) -//class SeawaterMonitor extends BaseBatch { -class SeawaterMonitor { +@BatchOwner(Watcher.ShenJian) +class SeawaterMonitor extends BaseBatch { static final Long expectNumber = 400000; // 运维组 运维警报 @@ -35,31 +37,7 @@ class SeawaterMonitor { static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] static void main(String[] args) { - def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") - def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") - def startDate = yesterday + "T00:00:00" - def endDate = yesterday + "T23:59:59" - - def result = [] - def msg = monitorYesterdayCrawlNumber(startDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - - msg = monitorCityCrawlNumber(startDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - - msg = monitorSiteCrawlNumber(lastWeekDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - if (result.size() > 0) { - msg = "【日期】 " + yesterday + "\n" + result.join("\n") - println(msg) - DingDingMessageUtil.sendTextMessage(access_token, msg) - } + runBatch(new SeawaterMonitor(), args) } /** @@ -141,8 +119,32 @@ class SeawaterMonitor { return "" } -// @Override -// void process(Object args) { -// -// } + @Override + void process(Object args) { + def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") + def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") + def startDate = yesterday + "T00:00:00" + def endDate = yesterday + "T23:59:59" + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } } From b8d282112291ed0de9bbb0cb1325b5c53753c5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 09:46:19 +0800 Subject: [PATCH 17/23] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=E5=8F=8A=E9=87=8D=E7=82=B9=E5=9F=8E=E5=B8=82=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index 60e2419de..8627efc19 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -7,7 +8,6 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import org.apache.commons.lang.StringUtils -import org.apache.commons.lang.time.DateUtils /** * 抓取总量监控 @@ -22,17 +22,17 @@ class SeawaterMonitor extends BaseBatch { // 运维组 运维警报 static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" // 重点城市 - static final city = ["北京','唐山','天津','秦皇岛','石家庄','包头','太原','丹东','呼和浩特','锦州'," + - "'沈阳','吉林','大连','牡丹江','长春','无锡','哈尔滨','扬州','上海','徐州'," + - "'南京','温州','杭州','金华','宁波','蚌埠','合肥','安庆','福州','泉州'," + - "'厦门','九江','南昌','赣州','济南','烟台','青岛','济宁','郑州','洛阳'," + - "'武汉','平顶山','长沙','宜昌','广州','襄阳','深圳','岳阳','南宁','常德'," + - "'海口','惠州','重庆','湛江','成都','韶关','贵阳','桂林','昆明','北海'," + - "'西安','三亚','兰州','泸州','西宁','南充','银川','遵义','乌鲁木齐','大理'," + - "'东莞','佛山','苏州','珠海','中山','台州','绍兴','南通','常州','嘉兴'," + - "'淄博','株洲','镇江','肇庆','漳州','盐城','芜湖','泰州','汕头','清远'," + - "'马鞍山','柳州','连云港','廊坊','淮安','湖州','沧州','保定','张家口','日照'," + - "'拉萨','大同','文昌','陵水','保亭"]; + static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州", + "沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州", + "南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州", + "厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳", + "武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德", + "海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海", + "西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理", + "东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴", + "淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远", + "马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照", + "拉萨","大同","文昌","陵水","保亭"]; // 重点网站 static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] @@ -46,7 +46,7 @@ class SeawaterMonitor extends BaseBatch { */ static def monitorYesterdayCrawlNumber(def startDate, def endDate) { def total - String sql = "select count(*) as total from latest_seawater where m_date between '${startDate}' and '${endDate}' " + String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' " ElasticUtils.executeSqlQuery(sql,{data-> total = data.total }) @@ -65,7 +65,7 @@ class SeawaterMonitor extends BaseBatch { result << ['城市', '抓取量'] // 统计近半个月各重点城市抓取量 def cityFormatted = city.join("','") - String sql = "select * from (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " ElasticUtils.executeSqlQuery(sql,{data-> result << [data.city, data.total] }) @@ -78,14 +78,20 @@ class SeawaterMonitor extends BaseBatch { */ static def monitorCityCrawlNumber(def startDate, def endDate) { def msg = "【重点城市抓取异常】" - def unNormal = [] + def set = new HashSet() def cityFormatted = city.join("','") String sql = "select * from " + - " (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) where total=0 " + " (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 " //" order by total " ElasticUtils.executeSqlQuery(sql,{data-> - unNormal.add(data.city) + set.add(data.city) }) + def unNormal = [] + for (def info : city) { + if (!set.contains(info)) { + unNormal.add(info) + } + } msg += unNormal.join("、") + " 抓取量为0" if (unNormal.size() > 0) { return msg @@ -101,7 +107,7 @@ class SeawaterMonitor extends BaseBatch { def set = new HashSet() def siteFormatted = site.join("','") String sql = "select * from " + - " (select count(*) as total, root from latest_seawater where m_date between '${lastWeekDate}' and '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + " (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " //" order by total " ElasticUtils.executeSqlQuery(sql,{data-> set.add(data.root) @@ -121,10 +127,10 @@ class SeawaterMonitor extends BaseBatch { @Override void process(Object args) { - def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") - def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") - def startDate = yesterday + "T00:00:00" - def endDate = yesterday + "T23:59:59" + def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() + def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1)) + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime(); + def endDate = DateUtil.truncateDay(new Date()).getTime() def result = [] def msg = monitorYesterdayCrawlNumber(startDate, endDate) From c363a8d0caad1a16e31baebad27bbb78c250b429 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Tue, 18 May 2021 12:51:48 +0800 Subject: [PATCH 18/23] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E8=A1=97=E9=81=93?= =?UTF-8?q?=E5=AD=97=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/PropertyController.java | 2 +- .../com/fangjia/script/wenyan/Test4.groovy | 103 +++++++++++------- .../fangjia/script/wenyan/test/Test1.groovy | 5 + 3 files changed, 71 insertions(+), 39 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 93172390a..37b785212 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -671,7 +671,7 @@ public class PropertyController { && !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId) && !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId) && !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId) - && !"1007747".equals(userId) + && !"1007747".equals(userId) && !"1007757".equals(userId) && model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制 mav.setModel(BasicModel.buildUnAuthorizedModel()); return mav; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index 95d41237b..235253d60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.ProvinceData import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -11,45 +12,71 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) -def count = 0 -List> data = Collections.synchronizedList([]) -data << ["ID","城市","区域","小区名","经度","纬度","街道"] - -CommonUtils.findAllCity().each {def city-> - DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> - if(StringUtils.isBlank(item.township) && item.lng){ - count++ - def lng = item.lng/10000000000d - def lat = item.lat/10000000000d - def location = "${lng},${lat}" - JSONObject resp = GaoDeApiUtils.geocode(location) - def row = [] - row << item._id.toString() - row << item.city - row << item.region - row << item.name - row << lng - row << lat - BasicDBObject dbo = new BasicDBObject() - dbo.id = item._id.toString() - dbo.city = item.city - dbo.region = item.region - dbo.name = item.name - dbo.lng = lng - dbo.lat = lat - if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ - row << resp.regeocode.addressComponent.township - dbo.township = resp.regeocode.addressComponent.township - } - println(row) - data << row - SEAWEED_TOWNSHIP.insert(dbo) - } - }) -} -ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +//def count = 0 +//List> data = Collections.synchronizedList([]) +//data << ["ID","城市","区域","小区名","经度","纬度","街道"] +// +//CommonUtils.findAllCity().each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(StringUtils.isBlank(item.township) && item.lng){ +// count++ +// def lng = item.lng/10000000000d +// def lat = item.lat/10000000000d +// def location = "${lng},${lat}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// def row = [] +// row << item._id.toString() +// row << item.city +// row << item.region +// row << item.name +// row << lng +// row << lat +// BasicDBObject dbo = new BasicDBObject() +// dbo.id = item._id.toString() +// dbo.city = item.city +// dbo.region = item.region +// dbo.name = item.name +// dbo.lng = lng +// dbo.lat = lat +// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ +// row << resp.regeocode.addressComponent.township +// dbo.township = resp.regeocode.addressComponent.township +// } +// println(row) +// data << row +// SEAWEED_TOWNSHIP.insert(dbo) +// } +// }) +//} +//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item -> +// if(item["G"] && item["H"] == null){ +// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"]))) +// } +// println(item) +//}) +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item-> +// def location = "${item["A"]},${item["B"]}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// if(resp.regeocode && resp.regeocode.formatted_address){ +// item['G'] = resp.regeocode.formatted_address +// println(item) +// } +//}) +//def count = 0 +//CommonUtils.findAllCity().each {def city-> +// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(item.township){ +// count++ +// } +// }, new BasicDBObject("township", 1)) +//} +//println(count) +ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{ + println(it) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy index 401d647e4..ff32247d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -9,6 +9,11 @@ import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils import net.sf.json.JSONObject +def strt(String addr){ + +} + + ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> def city = item["A"] def address1 = item["B"] From f3220beef36f8af94c1ad1bfb5f0ed190473c6ee Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 2 Jun 2021 15:36:07 +0800 Subject: [PATCH 19/23] update --- .../com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java | 5 +++-- .../fangjia/open/model/PropertyEvaluationModel.java | 10 ++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 9ff57486c..10a5791e3 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -199,7 +199,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } private JSONObject query(String id,String category, String city, String region, String name, List alias, - List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){ + List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, + Integer room, String root,String sort, boolean order, int start, int limit,String table){ long time1 = System.currentTimeMillis(); JSONObject reqBody = new JSONObject(); JSONObject q = new JSONObject(); @@ -313,7 +314,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(endPrice != null){ temp2.put("lte", endPrice); } - param.put("total_price", temp2); + param.put("avg_price", temp2); temp.put("range", param); jsonList.add(temp); } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java index 61f0618f7..5953fc3a3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java @@ -284,6 +284,8 @@ public class PropertyEvaluationModel extends FangjiaApiModel { private String user; + private Integer avgPrice; + public void setDecorationAge(int decorationAge) { this.decorationAge = decorationAge; @@ -895,4 +897,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel { public void setAtticSize(Double atticSize) { this.atticSize = atticSize; } + + public Integer getAvgPrice() { + return avgPrice; + } + + public void setAvgPrice(Integer avgPrice) { + this.avgPrice = avgPrice; + } } From f3ab53f6ebab12f780a2a93f091f95b8b5d2f265 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 2 Jun 2021 16:27:44 +0800 Subject: [PATCH 20/23] =?UTF-8?q?=E4=BA=BA=E5=B7=A5=E5=AE=A1=E6=9F=A5?= =?UTF-8?q?=E5=9C=B0=E5=9D=80=20=E6=B7=BB=E5=8A=A0=E5=8C=BA=E5=9F=9F?= =?UTF-8?q?=E5=8F=8A=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/com/fangjia/open/controllers/GuzhiController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java index 2f3621338..c2c3840e0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java @@ -161,7 +161,7 @@ public class GuzhiController { RestfulModelAndView mv = new RestfulModelAndView(); mv.setModel(BasicModel.buildOKModel()); - String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity() + String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()+" 区域:"+model.getDistrict() +" 小区名称:"+(model.getName()!=null?model.getName():"") + " 产证地址:" + model.getAddress(); DingDingMessageUtil.sendTextMessage("065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60", msg); return mv; From 5d09b016db9d59de62aa836c5a106ea0ce22b87e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 2 Jun 2021 16:30:45 +0800 Subject: [PATCH 21/23] log --- .../main/java/com/fangjia/open/controllers/GuzhiController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java index c2c3840e0..870dda5e3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java @@ -156,7 +156,7 @@ public class GuzhiController { return mv; } - + //福州农商行人工录入 public Object checkAction(@Valid GuzhiCheckModel model){ RestfulModelAndView mv = new RestfulModelAndView(); From ab4bbbffcd1e36492105df80c2a04f9b2935f025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 7 Jun 2021 13:43:20 +0800 Subject: [PATCH 22/23] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=9B=91=E6=8E=A7?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- deliverables/k8s/fangjia-bkoff.yaml | 10 ++++--- .../bkoff/controllers/K8sController.java | 28 +++++++++++++++++++ .../bkoff/interceptor/AuthInterceptor.java | 3 +- 3 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java diff --git a/deliverables/k8s/fangjia-bkoff.yaml b/deliverables/k8s/fangjia-bkoff.yaml index a6e01a7ab..768fd80cd 100644 --- a/deliverables/k8s/fangjia-bkoff.yaml +++ b/deliverables/k8s/fangjia-bkoff.yaml @@ -39,13 +39,15 @@ spec: name: web protocol: TCP readinessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/readiness + port: web initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/liveness + port: web initialDelaySeconds: 10 periodSeconds: 5 volumeMounts: diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java new file mode 100644 index 000000000..11a80b8a2 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java @@ -0,0 +1,28 @@ +package com.fangjia.bkoff.controllers; + +/** + * 房价网后台存活监控 + * + * @author shenjian + * @since 2021/6/7 + */ +public class K8sController { + + /** + * 存活检查 + * + * @return + */ + public Object livenessAction() { + return "liveness"; + } + + /** + * 就绪检查 + * + * @return + */ + public Object readinessAction() { + return "readiness"; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java index 192767d85..111b3c515 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java @@ -40,7 +40,8 @@ public class AuthInterceptor implements Interceptor { request.setAttribute("__resources", request.getSession().getAttribute(userId + "_resources")); if (invocation.getAction().getController().getName().equals("login") - || invocation.getAction().getController().getName().equals("")) { + || invocation.getAction().getController().getName().equals("") + || invocation.getAction().getController().getName().equals("k8s")) { return invocation.invoke(); } From 2d0b90b9e552c037c58a27014216a09377ced340 Mon Sep 17 00:00:00 2001 From: xuhe Date: Sat, 19 Jun 2021 19:26:29 +0800 Subject: [PATCH 23/23] =?UTF-8?q?=E9=98=BF=E9=87=8C=E6=8E=A5=E5=8F=A3?= =?UTF-8?q?=E7=A6=81=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/AddressController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 96bbe097f..ffd9e0b9a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1768,7 +1768,7 @@ public class AddressController { matchParams.setProvince(model.getProvince()); // matchParams.setUseMap(true); - matchParams.setUseAliyunApi(true); + matchParams.setUseAliyunApi(false); // matchParams.setUseCrf(true); matchParams.setContainsLocation(model.isContainsLocation()); Building building = addressParserService.standard(matchParams);