diff --git a/deliverables/k8s/fangjia-bkoff.yaml b/deliverables/k8s/fangjia-bkoff.yaml index a6e01a7ab..768fd80cd 100644 --- a/deliverables/k8s/fangjia-bkoff.yaml +++ b/deliverables/k8s/fangjia-bkoff.yaml @@ -39,13 +39,15 @@ spec: name: web protocol: TCP readinessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/readiness + port: web initialDelaySeconds: 5 periodSeconds: 10 livenessProbe: - tcpSocket: - port: 8080 + httpGet: + path: /k8s/liveness + port: web initialDelaySeconds: 10 periodSeconds: 5 volumeMounts: diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java new file mode 100644 index 000000000..11a80b8a2 --- /dev/null +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/controllers/K8sController.java @@ -0,0 +1,28 @@ +package com.fangjia.bkoff.controllers; + +/** + * 房价网后台存活监控 + * + * @author shenjian + * @since 2021/6/7 + */ +public class K8sController { + + /** + * 存活检查 + * + * @return + */ + public Object livenessAction() { + return "liveness"; + } + + /** + * 就绪检查 + * + * @return + */ + public Object readinessAction() { + return "readiness"; + } +} diff --git a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java index 192767d85..111b3c515 100644 --- a/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java +++ b/projects/fangjia-bkoff/src/main/java/com/fangjia/bkoff/interceptor/AuthInterceptor.java @@ -40,7 +40,8 @@ public class AuthInterceptor implements Interceptor { request.setAttribute("__resources", request.getSession().getAttribute(userId + "_resources")); if (invocation.getAction().getController().getName().equals("login") - || invocation.getAction().getController().getName().equals("")) { + || invocation.getAction().getController().getName().equals("") + || invocation.getAction().getController().getName().equals("k8s")) { return invocation.invoke(); } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java index 3a6029b19..e71226a4a 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seawater/dao/SeawaterDaoImpl.java @@ -237,7 +237,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i } private JSONObject query(String id,String category, String city, String region, String name, List alias, - List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){ + List regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, + Integer room, String root,String sort, boolean order, int start, int limit,String table){ long time1 = System.currentTimeMillis(); JSONObject reqBody = new JSONObject(); JSONObject q = new JSONObject(); @@ -351,7 +352,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl i if(endPrice != null){ temp2.put("lte", endPrice); } - param.put("total_price", temp2); + param.put("avg_price", temp2); temp.put("range", param); jsonList.add(temp); } diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java index 401132cb0..afba53e15 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/seaweed/model/District.java @@ -122,10 +122,8 @@ public class District implements SeaweedEntity { private String houseStructure; //房屋结构 private String source; private String propertyLife; -// -// //乡镇街道 -// private String township; -// + //乡镇街道 + private String township; private String pinyin; @@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity { this.mappingAlias = mappingAlias; } -// @Column(name = "township") -// public String getTownship() { -// return township; -// } -// -// public void setTownship(String township) { -// this.township = township; -// } + @Column(name = "township") + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index c9973367c..ffd9e0b9a 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1705,7 +1705,7 @@ public class AddressController { return mav; } - public Object addressSimilarityAction(AddesssMatchModel model) { + public Object similarityAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空")); @@ -1768,7 +1768,8 @@ public class AddressController { matchParams.setProvince(model.getProvince()); // matchParams.setUseMap(true); - matchParams.setUseAliyunApi(true); + matchParams.setUseAliyunApi(false); +// matchParams.setUseCrf(true); matchParams.setContainsLocation(model.isContainsLocation()); Building building = addressParserService.standard(matchParams); Address address = building.getSearchAddress(); diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java index 2f3621338..870dda5e3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/GuzhiController.java @@ -156,12 +156,12 @@ public class GuzhiController { return mv; } - + //福州农商行人工录入 public Object checkAction(@Valid GuzhiCheckModel model){ RestfulModelAndView mv = new RestfulModelAndView(); mv.setModel(BasicModel.buildOKModel()); - String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity() + String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()+" 区域:"+model.getDistrict() +" 小区名称:"+(model.getName()!=null?model.getName():"") + " 产证地址:" + model.getAddress(); DingDingMessageUtil.sendTextMessage("065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60", msg); return mv; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 93172390a..37b785212 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -671,7 +671,7 @@ public class PropertyController { && !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId) && !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId) && !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId) - && !"1007747".equals(userId) + && !"1007747".equals(userId) && !"1007757".equals(userId) && model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制 mav.setModel(BasicModel.buildUnAuthorizedModel()); return mav; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java index 3e97020e7..3a17ffc8b 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/CcbRentController.java @@ -9,6 +9,7 @@ import com.fangjia.dal.mongo.model.Evaluation; import com.fangjia.dal.mongo.model.RentDealPrice; import com.fangjia.dal.seawater.model.Seawater; import com.fangjia.dal.seaweed.model.District; +import com.fangjia.open.APIConstant; import com.fangjia.open.compatible.model.JsonModel; import com.fangjia.open.dto.DistrictHouseDtao; import com.fangjia.open.interceptors.AuthenticationBean; @@ -30,7 +31,6 @@ import org.apache.batik.transcoder.Transcoder; import org.apache.batik.transcoder.TranscoderException; import org.apache.batik.transcoder.TranscoderInput; import org.apache.batik.transcoder.TranscoderOutput; -import org.apache.batik.transcoder.image.JPEGTranscoder; import org.apache.batik.transcoder.image.PNGTranscoder; import org.apache.commons.lang.StringUtils; import org.apache.lucene.index.Term; @@ -46,10 +46,10 @@ import org.sothis.web.mvc.annotation.Param; import org.xhtmlrenderer.pdf.ITextFontResolver; import org.xhtmlrenderer.pdf.ITextRenderer; -import javax.servlet.ServletOutputStream; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; import java.io.*; +import java.lang.reflect.InvocationTargetException; import java.text.ParseException; import java.util.*; @@ -87,7 +87,26 @@ public class CcbRentController { return result; } - public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr, + public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException { + String result = null; + HttpRequest httpRequest = new HttpRequest(); + try { + Map headers = new HashMap(); + // &__refresh_cache=1 刷新缓存 + String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN; + Map params = org.apache.commons.beanutils.BeanUtils.describe(model); + result = httpRequest.post(url, params, headers,"UTF-8"); + } catch (NoSuchMethodException e) { + e.printStackTrace(); + } catch (HttpRequestException e) { + e.printStackTrace(); + } catch (InvocationTargetException e) { + e.printStackTrace(); + } + return result; + } + + public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr, @Param(name = "name") String name, @Param(name = "address") String address, @Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room, @Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) { diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java index 61f0618f7..5953fc3a3 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/PropertyEvaluationModel.java @@ -284,6 +284,8 @@ public class PropertyEvaluationModel extends FangjiaApiModel { private String user; + private Integer avgPrice; + public void setDecorationAge(int decorationAge) { this.decorationAge = decorationAge; @@ -895,4 +897,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel { public void setAtticSize(Double atticSize) { this.atticSize = atticSize; } + + public Integer getAvgPrice() { + return avgPrice; + } + + public void setAvgPrice(Integer avgPrice) { + this.avgPrice = avgPrice; + } } diff --git a/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl b/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl index 8633e3e89..60f3ac58f 100644 --- a/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl +++ b/projects/fangjia-open/src/main/webapp/ccbRent/index.ftl @@ -260,7 +260,7 @@ $("#evaluateBtn").prop("disabled", true) $.ajax({ // 娉ㄦ剰闇�瑕佸姞.. - url: '../property/evaluateRent', + url: '../ccbRent/evaluateRent', data:$("#evaluateForm").serialize(), dataType:"json", type:"post", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy index 85088ddd8..004d6bd4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy @@ -32,6 +32,7 @@ public enum Watcher { BaYi('15300424133','bayi@fangjia.com'), WeiLei('18101751850','weilei@fangjia.com'), JiangYaHui('13684913780','jiangyahui@fangjia.com'), + ShenJian('13764369683', 'shenjian@fangjia.com') private final String mobile; private final String email; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy new file mode 100644 index 000000000..8627efc19 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -0,0 +1,156 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import org.apache.commons.lang.StringUtils + +/** + * 抓取总量监控 + * + * @author shenjian + * @since 2021/5/12 + */ +@BatchOwner(Watcher.ShenJian) +class SeawaterMonitor extends BaseBatch { + + static final Long expectNumber = 400000; + // 运维组 运维警报 + static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + // 重点城市 + static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州", + "沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州", + "南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州", + "厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳", + "武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德", + "海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海", + "西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理", + "东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴", + "淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远", + "马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照", + "拉萨","大同","文昌","陵水","保亭"]; + // 重点网站 + static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] + + static void main(String[] args) { + runBatch(new SeawaterMonitor(), args) + } + + /** + * 监控前一日抓取量 + * @return + */ + static def monitorYesterdayCrawlNumber(def startDate, def endDate) { + def total + String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' " + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + if (total < expectNumber) { + return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber + } + return "" + } + + /** + * 统计重点城市近半月抓取量 + * @return + */ + static boolean countCityCrawlNumber(def startDate, def endDate) { + def result = [] + result << ['城市', '抓取量'] + // 统计近半个月各重点城市抓取量 + def cityFormatted = city.join("','") + String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + ElasticUtils.executeSqlQuery(sql,{data-> + result << [data.city, data.total] + }) + ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量") + } + + /** + * 统计前一日重点城市抓取量,昨天为零则警报 + * @return + */ + static def monitorCityCrawlNumber(def startDate, def endDate) { + def msg = "【重点城市抓取异常】" + def set = new HashSet() + def cityFormatted = city.join("','") + String sql = "select * from " + + " (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + set.add(data.city) + }) + def unNormal = [] + for (def info : city) { + if (!set.contains(info)) { + unNormal.add(info) + } + } + msg += unNormal.join("、") + " 抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + + /** + * 统计重点网站抓取情况,近一周为0则警报 + */ + static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) { + def msg = "【重点网站抓取异常】" + def set = new HashSet() + def siteFormatted = site.join("','") + String sql = "select * from " + + " (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + set.add(data.root) + }) + def unNormal = [] + for (def info : site) { + if (!set.contains(info)) { + unNormal.add(info) + } + } + msg += unNormal.join("、") + " 近一周抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + + @Override + void process(Object args) { + def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() + def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1)) + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime(); + def endDate = DateUtil.truncateDay(new Date()).getTime() + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy index bef30856c..235253d60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test4.groovy @@ -4,36 +4,79 @@ import com.fangjia.core.ProvinceData import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018) +//def count = 0 +//List> data = Collections.synchronizedList([]) +//data << ["ID","城市","区域","小区名","经度","纬度","街道"] +// +//CommonUtils.findAllCity().each {def city-> +// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(StringUtils.isBlank(item.township) && item.lng){ +// count++ +// def lng = item.lng/10000000000d +// def lat = item.lat/10000000000d +// def location = "${lng},${lat}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// def row = [] +// row << item._id.toString() +// row << item.city +// row << item.region +// row << item.name +// row << lng +// row << lat +// BasicDBObject dbo = new BasicDBObject() +// dbo.id = item._id.toString() +// dbo.city = item.city +// dbo.region = item.region +// dbo.name = item.name +// dbo.lng = lng +// dbo.lat = lat +// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){ +// row << resp.regeocode.addressComponent.township +// dbo.township = resp.regeocode.addressComponent.township +// } +// println(row) +// data << row +// SEAWEED_TOWNSHIP.insert(dbo) +// } +// }) +//} +//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据") +//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item -> +// if(item["G"] && item["H"] == null){ +// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"]))) +// } +// println(item) +//}) -//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\数据.xlsx",{List item-> -// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){ -// item["D"] = "存在" -// }else{ -// item["D"] = "不存在" +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item-> +// def location = "${item["A"]},${item["B"]}" +// JSONObject resp = GaoDeApiUtils.geocode(location) +// if(resp.regeocode && resp.regeocode.formatted_address){ +// item['G'] = resp.regeocode.formatted_address +// println(item) // } //}) +//def count = 0 +//CommonUtils.findAllCity().each {def city-> +// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item-> +// if(item.township){ +// count++ +// } +// }, new BasicDBObject("township", 1)) +//} +//println(count) -DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com") -ProvinceData.getAllCity("福建").each {def city-> - int count = 0 - def s = new TreeMap() - DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat-> - count ++ - int credit = (stat.credit * 100)/10 - s[credit] = s.get(credit, 0) + 1 - } - - def data = [] - data << city - data << count - for(int i=0; i <=10; i++){ - data << s.get(i, 0) - } - println(data.join(",")) -} +ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{ + println(it) +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy new file mode 100644 index 000000000..ff32247d7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/test/Test1.groovy @@ -0,0 +1,51 @@ +package com.fangjia.script.wenyan.test + +import com.aliyuncs.DefaultAcsClient +import com.aliyuncs.IAcsClient +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest +import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse +import com.aliyuncs.profile.DefaultProfile +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject + +def strt(String addr){ + +} + + +ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item-> + def city = item["A"] + def address1 = item["B"] + def address2 = item["C"] + HttpRequest http = new HttpRequest() + def params = new HashMap() + params["city"] = city + params["address_1"] = address1 + params["address_2"] = address2 + params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8") + JSONObject result = json.result + item["D"] = result?.level + item["E"] = result?.formattedAddress_1 + item["F"] = result?.formattedAddress_2 + item["G"] = result?.isSameAddress + DefaultProfile profile = DefaultProfile.getProfile( + "cn-hangzhou", + "LTAIdaj4eSO79gzz", + "xw6EsvqydZ9HC6zAvIB3ruQqVHynxf"); + IAcsClient client = new DefaultAcsClient(profile); + GetAddressSimilarityRequest request = new GetAddressSimilarityRequest(); + request.setServiceCode("addrp") + request.setAppKey("Xcgtyhupey0h") + request.setText("${address1};${address2}") + GetAddressSimilarityResponse response = client.getAcsResponse(request) + String data = response.getData() + json = JSONObject.fromObject(data) + item["H"] = json["sim_level"]["level"] + item["I"] = json["sim_level"]["address_a"] + item["J"] = json["sim_level"]["address_b"] + item["K"] = json["sim_level"]["is_same"] + println(json) +}) + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index 8bc805491..46545fc04 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -1,14 +1,12 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; -import com.fangjia.core.util.CNNMFilter; -import com.fangjia.core.util.DataUtil; -import com.fangjia.core.util.GeoUtils; -import com.fangjia.core.util.StringUtil; +import com.fangjia.core.util.*; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; import com.fangjia.dal.mongo.fangjia.model.ServerArea; +import com.fangjia.dal.mongo.model.AmapPoi; import com.fangjia.dal.mongo.model.bkoff.HouseLoudong; import com.fangjia.dal.mongo.model.bkoff.Loudong; import com.fangjia.dal.seaweed.dao.SeaweedDao; @@ -25,6 +23,7 @@ import com.fangjia.service.mongo.bkoff.HouseLoudongService; import com.fangjia.service.mongo.bkoff.LoudongService; import com.google.common.base.Function; import com.google.common.collect.MapMaker; +import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import org.bson.types.ObjectId; @@ -47,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService { private LoudongService loudongService; private SeaweedDao seaweedDao; private AliyunService aliyunService; - private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"}; + private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"}; private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class); @@ -134,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService { formatAddress(building); } return building; + /*JSONObject result = getDataFromCRF("test"); + return new Building();*/ } @Override @@ -165,6 +166,21 @@ public class AddressParserServiceImpl implements AddressParserService { return building; } Building matchedBuilding; + //调用CRF算法切分数据 + /*JSONObject jsonObj = null; + if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + jsonObj = getDataFromCRF(addr.getOrgAddress()); + if(jsonObj != null){ + String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") + + (jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") + + (jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : ""); + if(StringUtils.isNotBlank(poi)){ + if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){ + addr.setKeywords(poi); + } + } + } + }*/ //keywords使用aliyun接口返回的poi AliyunBo aliyunBo = null; if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ @@ -176,6 +192,9 @@ public class AddressParserServiceImpl implements AddressParserService { addr.setKeywords(poi); } } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){ String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length())); if(StringUtils.isNotBlank(str)){ @@ -187,26 +206,80 @@ public class AddressParserServiceImpl implements AddressParserService { } } } - if(StringUtils.isNotBlank(aliyunBo.getCity())){ - addr.setSuggestCity(aliyunBo.getCity()); - } - } - } - AddressMatchMetaData meta = null; - if(StringUtils.isBlank(addr.getCity())){ - addr.addKeyword(addr.getOrgAddress()); - meta = getMapMeta(addr); - if(meta != null){ - addr.setProvince(meta.getMapProvince()); - addr.setCity(meta.getMapCity()); - addr.setDistrict(meta.getDistrict()); } } + addr.addKeyword(addr.getOrgAddress()); if(params.isUseMap()){ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP); }else{ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB); } + + /*if(jsonObj != null){ + //乡镇优化 + String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : ""; + if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){ + String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township); + if(StringUtils.isNotBlank(town)){ + addr.setTownship(town); + } + } + //道路优化(两边均去掉乡镇) + String street = addr.getStreet(); + String streetNumber = addr.getStreetNumber(); + String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : ""; + String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : ""; + //去掉路名中含有乡镇名的 + if(StringUtils.isNotBlank(addr.getTownship())){ + street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street; + streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf; + } + + //crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理 + if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){ + //Java解析出路名为空 + if(StringUtils.isBlank(street)){ + street = streetCrf; + if(StringUtils.isNotBlank(streetNumberCrf)){ + streetNumber = streetNumberCrf; + } + }else{ + //CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则 + if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){ + street = streetCrf; + } + } + addr.setStreet(street); + addr.setStreetNumber(streetNumber); + } + //poi处理 + //没有匹配上库中小区时 + if(StringUtils.isBlank(matchedBuilding.getId())){ + String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") + + (jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") + + (jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : ""); + //去掉路名中含有乡镇名和路名 getParsedPropertyName + if(StringUtils.isNotBlank(addr.getTownship())){ + poi = poi.replace(addr.getTownship(),""); + } + if(StringUtils.isNotBlank(addr.getStreet())){ + poi = poi.replace(addr.getStreet(),""); + } + addr.setParsedPropertyName(poi); + } + //楼栋、单元、室号 + String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", ""); + addr.setOrgBuildingNumber(houseNo); + addr.setBuildingNumber(houseNo); + addr.setOrgUnitNumber(cellNo); + addr.setUnitNumber(cellNo); + addr.setFloor(floorNo); + addr.setOrgRoomNumber(roomNo); + addr.setRoomNumber(roomNo); + }*/ matchedBuilding.setProvince(addr.getProvince()); matchedBuilding.setSearchAddress(addr); matchedBuilding.setSuggestName(getSuggestName(matchedBuilding)); @@ -215,7 +288,8 @@ public class AddressParserServiceImpl implements AddressParserService { completBuildingField(matchedBuilding, aliyunBo, params); } Address tempAddr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(tempAddr.getStreet()) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){ + AddressMatchMetaData meta = getMapMeta(addr); if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){ matchedBuilding.setAddress(meta.getAddress()); } @@ -255,6 +329,25 @@ public class AddressParserServiceImpl implements AddressParserService { } return null; } + + /** + * TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用 + * @param address + * @return + */ + /*private JSONObject getDataFromCRF(String address){ + HttpRequest request = new HttpRequest(); + Map params = new HashMap<>(); + params.put("address",address); + try { + JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8"); + System.out.println(resp); + return resp; + }catch(Exception e){ + + } + return null; + }*/ /** * 信息不全时使用阿里接口补全数据 * @param matchedBuilding @@ -271,33 +364,44 @@ public class AddressParserServiceImpl implements AddressParserService { } } Address addr = matchedBuilding.getSearchAddress(); - if(StringUtils.isBlank(addr.getProvince())){ + if(StringUtils.isNotBlank(aliyunBo.getProv())){ + addr.setProvince(formatProvince(aliyunBo.getProv())); + } + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } + if(StringUtils.isNotBlank(aliyunBo.getDistrict())){ + addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); + } + addr.setTownship(aliyunBo.getTown()); + + /*if(StringUtils.isBlank(addr.getProvince())){ addr.setProvince(formatProvince(aliyunBo.getProv())); } if(StringUtils.isBlank(addr.getCity())){ addr.setCity(RegionData.formatCity(aliyunBo.getCity())); } - - if(StringUtils.isBlank(addr.getDevelopmentZone())){ - addr.setDevelopmentZone(aliyunBo.getDevzone()); - } if(StringUtils.isBlank(addr.getDistrict())){ addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); } if(StringUtils.isBlank(addr.getTownship())){ addr.setTownship(aliyunBo.getTown()); + }*/ + if(StringUtils.isBlank(addr.getDevelopmentZone())){ + addr.setDevelopmentZone(aliyunBo.getDevzone()); } + /*addr.setOrgBuildingNumber(aliyunBo.getHouseno()); addr.setBuildingNumber(aliyunBo.getHouseno()); addr.setOrgUnitNumber(aliyunBo.getCellno()); addr.setUnitNumber(aliyunBo.getCellno());*/ - if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ - addr.setRoomNumber(aliyunBo.getRoomno()); + /*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){ + addr.setOrgRoomNumber(aliyunBo.getRoomno()); addr.setRoomNumber(aliyunBo.getRoomno()); }else{ addr.setRoomNumber(null); addr.setRoomNumber(null); - } + }*/ /*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", ""); String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", ""); @@ -1037,20 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService { if(StringUtils.isNotBlank(address.getTownship())){ formattedAddressBuilder.append(address.getTownship()); } + if(StringUtils.isNotBlank(building.getAddress())){ + Address orgAddr = new Address(); + orgAddr.setOrgAddress(building.getAddress()); + Address addr = AddressProcessors.process(orgAddr); + if(StringUtils.isBlank(address.getStreet())){ + address.setStreet(addr.getStreet()); + } + if(StringUtils.isBlank(address.getStreetNumber())){ + address.setStreetNumber(addr.getStreetNumber()); + } + } if(StringUtils.isNotBlank(address.getStreet())){ formattedAddressBuilder.append(address.getStreet()); } if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){ formattedAddressBuilder.append(address.getStreetNumber()); } - if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ - Address orgAddr = new Address(); - orgAddr.setOrgAddress(building.getAddress()); - Address addr = AddressProcessors.process(orgAddr); - building.getSearchAddress().setStreet(addr.getStreet()); - building.getSearchAddress().setStreetNumber(addr.getStreetNumber()); - formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber())); - } + if(StringUtils.isNotBlank(building.getSuggestName())){ formattedAddressBuilder.append(building.getSuggestName()); } @@ -1077,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService { formattedAddressBuilder.append(address.getUnitNumber() + "单元"); } if(StringUtils.isNotBlank(address.getRoomNumber())){ - formattedAddressBuilder.append(address.getRoomNumber()); -// formattedAddressBuilder.append(address.getRoomNumber() + "室"); +// formattedAddressBuilder.append(address.getRoomNumber()); + formattedAddressBuilder.append(address.getRoomNumber() + "室"); } address.setFormattedAddress(formattedAddressBuilder.toString()); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 58022cf03..1c8f2d004 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -77,6 +77,16 @@ public class Address { private String debugName; private AliyunBo aliyunBo; + //使用CRF切分的地址 + private String crfAddress; + + public String getCrfAddress() { + return crfAddress; + } + + public void setCrfAddress(String crfAddress) { + this.crfAddress = crfAddress; + } public AddressParserGoal getAddressParserGoal() { return addressParserGoal; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java index a217fa8fd..251470ffc 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java @@ -64,9 +64,7 @@ public class Building { this.alias = d.getAlias(); this.addrAlias = d.getAddrAlias(); this.propertyType = d.getEstateType2(); - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); if(d.getLat() != null && d.getLng() != null){ this.lat = d.getLat()/10000000000d; this.lng = d.getLng()/10000000000d; @@ -90,9 +88,7 @@ public class Building { this.lat = d.getLat2()/10000000000d; this.lng = d.getLng2()/10000000000d; } - if(StringUtils.isNotBlank(d.getStreetCommunity())){ - this.township = d.getStreetCommunity(); - } + this.township = d.getTownship(); } public Building() { diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java index 498b12052..e53c9701f 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java @@ -25,7 +25,15 @@ public class MatchParams { private boolean useMap; /**是否使用阿里云接口*/ private boolean useAliyunApi; + //是否使用CRF算法 + private boolean useCrf; + public boolean isUseCrf() { + return useCrf; + } + public void setUseCrf(boolean useCrf) { + this.useCrf = useCrf; + } public boolean isContainsLocation() { return containsLocation; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java index 93c69b4c4..ab5c929b3 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/StageAndPartitionProcessor.java @@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor { @Override public RepeatStatus process(Address address) { String orgAddr = address.getEffectiveAddress(); + address.setCrfAddress(address.getAddress()); MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress()); if (matchResult != null) { String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java index b6b22f5a1..91d309485 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -1,5 +1,7 @@ package com.fangjia.service.api; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest; +import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse; import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest; import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse; import com.fangjia.core.util.JsonUtils; @@ -13,6 +15,9 @@ import net.sf.json.JSONObject; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; +import java.util.ArrayList; +import java.util.List; + public class AliyunServiceImpl implements AliyunService { protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); @@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService { if(StringUtils.isNotBlank(city)){ request.setDefaultCity(city); } + AliyunBo aliyunBo = new AliyunBo(); try{ StructureAddressResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); String value = (String) responseJson.get("structure"); - AliyunBo aliyunBo = new AliyunBo(); for (String key:value.split("\t")){ String[] arry = key.split("="); wrapField(arry[0],arry[1], aliyunBo); } - return aliyunBo; }catch(Exception e){ - LOGGER.error("", e); - return null; + LOGGER.error("调用阿里标准化接口失败:", e); } + try{ + GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest(); + divisionRequest.setServiceCode("addrp"); + divisionRequest.setAppKey(APPKEY); + divisionRequest.setText(address); + GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest); + JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData()); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + aliyunBo.setProv(array[0]); + aliyunBo.setCity(array[1]); + aliyunBo.setDistrict(array[2]); + aliyunBo.setTown(array[3]); + }catch(Exception e){ + LOGGER.error("调用阿里行政区划识别接口失败", e); + } + return aliyunBo; } private void wrapField(String field,String value, AliyunBo aliyunBo){ @@ -81,26 +110,59 @@ public class AliyunServiceImpl implements AliyunService { } } public static void main(String[] args) throws ClientException { - DefaultProfile profile = DefaultProfile.getProfile( + String value = ""; + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]); + + /*DefaultProfile profile = DefaultProfile.getProfile( REGIONID, ACCESSKEYID, SECRET); IAcsClient client = new DefaultAcsClient(profile); - StructureAddressRequest request = new StructureAddressRequest(); +// StructureAddressRequest request = new StructureAddressRequest(); + GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest(); request.setServiceCode("addrp"); request.setAppKey(APPKEY); - request.setText("杭州市文一西路969号"); - request.setDefaultProvince("浙江省"); - request.setDefaultCity("杭州市"); - request.setDefaultDistrict("余杭区"); - StructureAddressResponse response = client.getAcsResponse(request); + request.setText("天青路260号"); +// request.setDefaultProvince("浙江省"); +// request.setDefaultCity("杭州市"); +// request.setDefaultDistrict("余杭区"); +// StructureAddressResponse response = client.getAcsResponse(request); + GetAddressDivisionCodeResponse response = client.getAcsResponse(request); JSONObject responseJson = JSONObject.fromObject(response.getData()); - String value = (String) responseJson.get("structure"); - JSONObject json = new JSONObject(); + System.out.println(responseJson); + String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name"); + String[] array = value.replace("\t", ";").split(";"); + if(array.length > 4){ + List tmp = new ArrayList(); + for(String str:array){ + if(str!=null && str.length()!=0){ + tmp.add(str); + } + } + array= tmp.toArray(new String[4]); + } + System.out.println(array[0]); + System.out.println(array[1]); + System.out.println(array[2]); + System.out.println(array[3]);*/ + /*JSONObject json = new JSONObject(); for (String key:value.split("\t")){ String[] arry = key.split("="); json.put(arry[0],arry[1]); } - System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class)); + System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/ } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java index 221fecc5b..21ab2139d 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/web/AddressServiceImpl.java @@ -5,18 +5,7 @@ import java.math.BigDecimal; import java.math.RoundingMode; import java.net.URLEncoder; import java.text.DecimalFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Comparator; -import java.util.Date; -import java.util.HashMap; -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; @@ -5080,7 +5069,7 @@ public class AddressServiceImpl implements AddressService { double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin); String level = getAddressSimilarityLevel(json1, json2); resultMap.put("isSameAddress", sameAddress); - resultMap.put("score", similar); +// resultMap.put("score", similar); resultMap.put("level", level); resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress")); resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress")); @@ -5138,30 +5127,51 @@ public class AddressServiceImpl implements AddressService { return null; } /** - * 获取两地址的相似度评级(最小颗粒度) + * 获取两地址的相似度评级(相同数据的最小颗粒度) * @return */ private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){ - if(StringUtils.isNotBlank(getFromJson(json1,"roomNumber")) || StringUtils.isNotBlank(getFromJson(json2,"roomNumber"))){ - return "roomNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"unitNumber")) || StringUtils.isNotBlank(getFromJson(json2,"unitNumber"))){ - return "unitNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"buildingNumber")) || StringUtils.isNotBlank(getFromJson(json2,"buildingNumber"))){ - return "buildingNumber"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"name")) || StringUtils.isNotBlank(getFromJson(json2,"name"))){ - return "name"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"street")) || StringUtils.isNotBlank(getFromJson(json2,"street"))){ - return "street"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"township")) || StringUtils.isNotBlank(getFromJson(json2,"township"))){ - return "township"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"district")) || StringUtils.isNotBlank(getFromJson(json2,"district"))){ - return "district"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"city")) || StringUtils.isNotBlank(getFromJson(json2,"city"))){ - return "city"; - }else if(StringUtils.isNotBlank(getFromJson(json1,"province")) || StringUtils.isNotBlank(getFromJson(json2,"province"))){ - return "province"; + String level = null; + LinkedList list = new LinkedList(Arrays.asList("province", "city", "district", "township", "street", "streetNumber" + , "name", "buildingNumber", "unitNumber", "roomNumber")); + for(String key:list){ + if(continueJudgeValue(json1, json2, key) == 3){ + return level; + }else{ + level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level); + } } - return null; + return level; + } + + /** + * 判断level时是否继续往下判断 + * @param json1 + * @param json2 + * @param key + * @return 1 继续判断并且level有值 2继续判断level没值 3 停止 + */ + private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){ + if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){ + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key)) + && getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } + if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){ + if("name".equals(key)){ + return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3; + }else{ + if(getFromJson(json1,key).equals(getFromJson(json2,key))){ + return 1; + }else{ + return 3; + } + } + } + return 2; } /** * 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true) @@ -5210,6 +5220,10 @@ public class AddressServiceImpl implements AddressService { if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){ return false; } + return isSameName(name1, name2); + } + + private boolean isSameName(String name1, String name2){ String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1); String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2); //拼音或文本数据包含并且 @@ -5222,7 +5236,6 @@ public class AddressServiceImpl implements AddressService { } return false; } - /** * 设置区域消费指数 * @param checkTr