Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj

This commit is contained in:
2021-07-05 16:41:13 +08:00
22 changed files with 651 additions and 142 deletions
+6 -4
View File
@@ -39,13 +39,15 @@ spec:
name: web
protocol: TCP
readinessProbe:
tcpSocket:
port: 8080
httpGet:
path: /k8s/readiness
port: web
initialDelaySeconds: 5
periodSeconds: 10
livenessProbe:
tcpSocket:
port: 8080
httpGet:
path: /k8s/liveness
port: web
initialDelaySeconds: 10
periodSeconds: 5
volumeMounts:
@@ -0,0 +1,28 @@
package com.fangjia.bkoff.controllers;
/**
* 房价网后台存活监控
*
* @author shenjian
* @since 2021/6/7
*/
public class K8sController {
/**
* 存活检查
*
* @return
*/
public Object livenessAction() {
return "liveness";
}
/**
* 就绪检查
*
* @return
*/
public Object readinessAction() {
return "readiness";
}
}
@@ -40,7 +40,8 @@ public class AuthInterceptor implements Interceptor {
request.setAttribute("__resources", request.getSession().getAttribute(userId + "_resources"));
if (invocation.getAction().getController().getName().equals("login")
|| invocation.getAction().getController().getName().equals("")) {
|| invocation.getAction().getController().getName().equals("")
|| invocation.getAction().getController().getName().equals("k8s")) {
return invocation.invoke();
}
@@ -237,7 +237,8 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
}
private JSONObject query(String id,String category, String city, String region, String name, List<String> alias,
List<String> regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice, Integer room, String root,String sort, boolean order, int start, int limit,String table){
List<String> regionAlias, Integer startDate, Integer endDate, Double startArea, Double endArea, Integer startPrice, Integer endPrice,
Integer room, String root,String sort, boolean order, int start, int limit,String table){
long time1 = System.currentTimeMillis();
JSONObject reqBody = new JSONObject();
JSONObject q = new JSONObject();
@@ -351,7 +352,7 @@ public class SeawaterDaoImpl extends com.fangjia.dal.SeawaterDaoImpl<Seawater> i
if(endPrice != null){
temp2.put("lte", endPrice);
}
param.put("total_price", temp2);
param.put("avg_price", temp2);
temp.put("range", param);
jsonList.add(temp);
}
@@ -122,10 +122,8 @@ public class District implements SeaweedEntity {
private String houseStructure; //房屋结构
private String source;
private String propertyLife;
//
// //乡镇街道
// private String township;
//
//乡镇街道
private String township;
private String pinyin;
@@ -1873,14 +1871,14 @@ public class District implements SeaweedEntity {
this.mappingAlias = mappingAlias;
}
// @Column(name = "township")
// public String getTownship() {
// return township;
// }
//
// public void setTownship(String township) {
// this.township = township;
// }
@Column(name = "township")
public String getTownship() {
return township;
}
public void setTownship(String township) {
this.township = township;
}
@@ -1705,7 +1705,7 @@ public class AddressController {
return mav;
}
public Object addressSimilarityAction(AddesssMatchModel model) {
public Object similarityAction(AddesssMatchModel model) {
RestfulModelAndView mav = new RestfulModelAndView();
if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) {
mav.setModel(BasicModel.buildBadRequestModel("两个地址均不能为空"));
@@ -1768,7 +1768,8 @@ public class AddressController {
matchParams.setProvince(model.getProvince());
// matchParams.setUseMap(true);
matchParams.setUseAliyunApi(true);
matchParams.setUseAliyunApi(false);
// matchParams.setUseCrf(true);
matchParams.setContainsLocation(model.isContainsLocation());
Building building = addressParserService.standard(matchParams);
Address address = building.getSearchAddress();
@@ -156,12 +156,12 @@ public class GuzhiController {
return mv;
}
//福州农商行人工录入
public Object checkAction(@Valid GuzhiCheckModel model){
RestfulModelAndView mv = new RestfulModelAndView();
mv.setModel(BasicModel.buildOKModel());
String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()
String msg = "【房价网-人工审查地址】接收到新的人工审查地址信息,请求请及时处理!城市:" + model.getCity()+" 区域:"+model.getDistrict() +" 小区名称:"+(model.getName()!=null?model.getName():"")
+ " 产证地址:" + model.getAddress();
DingDingMessageUtil.sendTextMessage("065f4a05d5f87eb39a9f9e048eeefb80c1f7560446f8926441de80453c160c60", msg);
return mv;
@@ -671,7 +671,7 @@ public class PropertyController {
&& !"1007486".equals(userId) && !"1007533".equals(userId) && !"1007630".equals(userId)
&& !"1007705".equals(userId) && !"1007739".equals(userId) && !"1007736".equals(userId)
&& !"1007741".equals(userId) && !"1007742".equals(userId) && !"1007743".equals(userId)
&& !"1007747".equals(userId)
&& !"1007747".equals(userId) && !"1007757".equals(userId)
&& model.getDetailLevel() != null && model.getDetailLevel() != 0) {// 临时权限控制
mav.setModel(BasicModel.buildUnAuthorizedModel());
return mav;
@@ -9,6 +9,7 @@ import com.fangjia.dal.mongo.model.Evaluation;
import com.fangjia.dal.mongo.model.RentDealPrice;
import com.fangjia.dal.seawater.model.Seawater;
import com.fangjia.dal.seaweed.model.District;
import com.fangjia.open.APIConstant;
import com.fangjia.open.compatible.model.JsonModel;
import com.fangjia.open.dto.DistrictHouseDtao;
import com.fangjia.open.interceptors.AuthenticationBean;
@@ -30,7 +31,6 @@ import org.apache.batik.transcoder.Transcoder;
import org.apache.batik.transcoder.TranscoderException;
import org.apache.batik.transcoder.TranscoderInput;
import org.apache.batik.transcoder.TranscoderOutput;
import org.apache.batik.transcoder.image.JPEGTranscoder;
import org.apache.batik.transcoder.image.PNGTranscoder;
import org.apache.commons.lang.StringUtils;
import org.apache.lucene.index.Term;
@@ -46,10 +46,10 @@ import org.sothis.web.mvc.annotation.Param;
import org.xhtmlrenderer.pdf.ITextFontResolver;
import org.xhtmlrenderer.pdf.ITextRenderer;
import javax.servlet.ServletOutputStream;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.io.*;
import java.lang.reflect.InvocationTargetException;
import java.text.ParseException;
import java.util.*;
@@ -87,7 +87,26 @@ public class CcbRentController {
return result;
}
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
public Object evaluateRentAction(com.fangjia.open.model.PropertyEvaluationModel model, HttpServletRequest request)throws NumberFormatException, IllegalAccessException {
String result = null;
HttpRequest httpRequest = new HttpRequest();
try {
Map<String, String> headers = new HashMap<String, String>();
// &__refresh_cache=1 ˢлº´æ
String url = "http://open.fangjia.com/property/evaluateRent?token=" + APIConstant.SUPER_USER_TOKEN;
Map<String, String> params = org.apache.commons.beanutils.BeanUtils.describe(model);
result = httpRequest.post(url, params, headers,"UTF-8");
} catch (NoSuchMethodException e) {
e.printStackTrace();
} catch (HttpRequestException e) {
e.printStackTrace();
} catch (InvocationTargetException e) {
e.printStackTrace();
}
return result;
}
public Object downloadPageAction(PropertyEvaluationModel result, HttpServletRequest request, @Param(name = "city") String city, @Param(name = "district") String districtStr,
@Param(name = "name") String name, @Param(name = "address") String address,
@Param(name = "size") Double size, @Param(name = "floor") Integer floor, @Param(name = "height") Integer height, @Param(name = "room") Integer room,
@Param(name = "hall") Integer hall, @Param(name = "toilet") Integer toilet, @Param(name = "direction") String direction, @Param(name = "fitment") String fitment, @Param(name = "chartImgUrl") String chartImgUrl) {
@@ -284,6 +284,8 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
private String user;
private Integer avgPrice;
public void setDecorationAge(int decorationAge) {
this.decorationAge = decorationAge;
@@ -895,4 +897,12 @@ public class PropertyEvaluationModel extends FangjiaApiModel {
public void setAtticSize(Double atticSize) {
this.atticSize = atticSize;
}
public Integer getAvgPrice() {
return avgPrice;
}
public void setAvgPrice(Integer avgPrice) {
this.avgPrice = avgPrice;
}
}
@@ -260,7 +260,7 @@
$("#evaluateBtn").prop("disabled", true)
$.ajax({
// 注意需要加..
url: '../property/evaluateRent',
url: '../ccbRent/evaluateRent',
data:$("#evaluateForm").serialize(),
dataType:"json",
type:"post",
@@ -32,6 +32,7 @@ public enum Watcher {
BaYi('15300424133','bayi@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com')
private final String mobile;
private final String email;
@@ -0,0 +1,156 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import org.apache.commons.lang.StringUtils
/**
* 抓取总量监控
*
* @author shenjian
* @since 2021/5/12
*/
@BatchOwner(Watcher.ShenJian)
class SeawaterMonitor extends BaseBatch {
static final Long expectNumber = 400000;
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
// 重点城市
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
"拉萨","大同","文昌","陵水","保亭"];
// 重点网站
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
static void main(String[] args) {
runBatch(new SeawaterMonitor(), args)
}
/**
* 监控前一日抓取量
* @return
*/
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
def total
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
if (total < expectNumber) {
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
}
return ""
}
/**
* 统计重点城市近半月抓取量
* @return
*/
static boolean countCityCrawlNumber(def startDate, def endDate) {
def result = []
result << ['城市', '抓取量']
// 统计近半个月各重点城市抓取量
def cityFormatted = city.join("','")
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
ElasticUtils.executeSqlQuery(sql,{data->
result << [data.city, data.total]
})
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
}
/**
* 统计前一日重点城市抓取量,昨天为零则警报
* @return
*/
static def monitorCityCrawlNumber(def startDate, def endDate) {
def msg = "【重点城市抓取异常】"
def set = new HashSet()
def cityFormatted = city.join("','")
String sql = "select * from " +
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.city)
})
def unNormal = []
for (def info : city) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
/**
* 统计重点网站抓取情况,近一周为0则警报
*/
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
def msg = "【重点网站抓取异常】"
def set = new HashSet()
def siteFormatted = site.join("','")
String sql = "select * from " +
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.root)
})
def unNormal = []
for (def info : site) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 近一周抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
@Override
void process(Object args) {
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
def endDate = DateUtil.truncateDay(new Date()).getTime()
def result = []
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorCityCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
if (result.size() > 0) {
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
println(msg)
DingDingMessageUtil.sendTextMessage(access_token, msg)
}
}
}
@@ -4,36 +4,79 @@ import com.fangjia.core.ProvinceData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
//def count = 0
//List<List<String>> data = Collections.synchronizedList([])
//data << ["ID","城市","区域","小区名","经度","纬度","街道"]
//
//CommonUtils.findAllCity().each {def city->
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
// if(StringUtils.isBlank(item.township) && item.lng){
// count++
// def lng = item.lng/10000000000d
// def lat = item.lat/10000000000d
// def location = "${lng},${lat}"
// JSONObject resp = GaoDeApiUtils.geocode(location)
// def row = []
// row << item._id.toString()
// row << item.city
// row << item.region
// row << item.name
// row << lng
// row << lat
// BasicDBObject dbo = new BasicDBObject()
// dbo.id = item._id.toString()
// dbo.city = item.city
// dbo.region = item.region
// dbo.name = item.name
// dbo.lng = lng
// dbo.lat = lat
// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
// row << resp.regeocode.addressComponent.township
// dbo.township = resp.regeocode.addressComponent.township
// }
// println(row)
// data << row
// SEAWEED_TOWNSHIP.insert(dbo)
// }
// })
//}
//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item ->
// if(item["G"] && item["H"] == null){
// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"])))
// }
// println(item)
//})
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Êý¾Ý.xlsx",{List item->
// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){
// item["D"] = "´æÔÚ"
// }else{
// item["D"] = "²»´æÔÚ"
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item->
// def location = "${item["A"]},${item["B"]}"
// JSONObject resp = GaoDeApiUtils.geocode(location)
// if(resp.regeocode && resp.regeocode.formatted_address){
// item['G'] = resp.regeocode.formatted_address
// println(item)
// }
//})
//def count = 0
//CommonUtils.findAllCity().each {def city->
// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
// if(item.township){
// count++
// }
// }, new BasicDBObject("township", 1))
//}
//println(count)
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com")
ProvinceData.getAllCity("¸£½¨").each {def city->
int count = 0
def s = new TreeMap()
DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat->
count ++
int credit = (stat.credit * 100)/10
s[credit] = s.get(credit, 0) + 1
}
def data = []
data << city
data << count
for(int i=0; i <=10; i++){
data << s.get(i, 0)
}
println(data.join(","))
}
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
println(it)
})
@@ -0,0 +1,51 @@
package com.fangjia.script.wenyan.test
import com.aliyuncs.DefaultAcsClient
import com.aliyuncs.IAcsClient
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
import com.aliyuncs.profile.DefaultProfile
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.ExcelUtils
import net.sf.json.JSONObject
def strt(String addr){
}
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
def city = item["A"]
def address1 = item["B"]
def address2 = item["C"]
HttpRequest http = new HttpRequest()
def params = new HashMap()
params["city"] = city
params["address_1"] = address1
params["address_2"] = address2
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
JSONObject result = json.result
item["D"] = result?.level
item["E"] = result?.formattedAddress_1
item["F"] = result?.formattedAddress_2
item["G"] = result?.isSameAddress
DefaultProfile profile = DefaultProfile.getProfile(
"cn-hangzhou",
"LTAIdaj4eSO79gzz",
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
IAcsClient client = new DefaultAcsClient(profile);
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
request.setServiceCode("addrp")
request.setAppKey("Xcgtyhupey0h")
request.setText("${address1};${address2}")
GetAddressSimilarityResponse response = client.getAcsResponse(request)
String data = response.getData()
json = JSONObject.fromObject(data)
item["H"] = json["sim_level"]["level"]
item["I"] = json["sim_level"]["address_a"]
item["J"] = json["sim_level"]["address_b"]
item["K"] = json["sim_level"]["is_same"]
println(json)
})
@@ -1,14 +1,12 @@
package com.fangjia.service.address;
import com.fangjia.core.RegionData;
import com.fangjia.core.util.CNNMFilter;
import com.fangjia.core.util.DataUtil;
import com.fangjia.core.util.GeoUtils;
import com.fangjia.core.util.StringUtil;
import com.fangjia.core.util.*;
import com.fangjia.dal.QueryParam;
import com.fangjia.dal.common.DistrictKey;
import com.fangjia.dal.common.ServerAreaData;
import com.fangjia.dal.mongo.fangjia.model.ServerArea;
import com.fangjia.dal.mongo.model.AmapPoi;
import com.fangjia.dal.mongo.model.bkoff.HouseLoudong;
import com.fangjia.dal.mongo.model.bkoff.Loudong;
import com.fangjia.dal.seaweed.dao.SeaweedDao;
@@ -25,6 +23,7 @@ import com.fangjia.service.mongo.bkoff.HouseLoudongService;
import com.fangjia.service.mongo.bkoff.LoudongService;
import com.google.common.base.Function;
import com.google.common.collect.MapMaker;
import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import org.bson.types.ObjectId;
@@ -47,7 +46,7 @@ public class AddressParserServiceImpl implements AddressParserService {
private LoudongService loudongService;
private SeaweedDao seaweedDao;
private AliyunService aliyunService;
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"};
private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "township"};
private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class);
@@ -134,6 +133,8 @@ public class AddressParserServiceImpl implements AddressParserService {
formatAddress(building);
}
return building;
/*JSONObject result = getDataFromCRF("test");
return new Building();*/
}
@Override
@@ -165,6 +166,21 @@ public class AddressParserServiceImpl implements AddressParserService {
return building;
}
Building matchedBuilding;
//调用CRF算法切分数据
/*JSONObject jsonObj = null;
if(params.isUseCrf() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
jsonObj = getDataFromCRF(addr.getOrgAddress());
if(jsonObj != null){
String poi = (jsonObj.containsKey("community_model") ? jsonObj.getString("community_model") : "") +
(jsonObj.containsKey("poi_model") ? jsonObj.getString("poi_model") : "") +
(jsonObj.containsKey("subpoi_model") ? jsonObj.getString("subpoi_model") : "");
if(StringUtils.isNotBlank(poi)){
if(addr.getKeywords()==null || !addr.getKeywords().contains(poi)){
addr.setKeywords(poi);
}
}
}
}*/
//keywords使用aliyun接口返回的poi
AliyunBo aliyunBo = null;
if(params.isUseAliyunApi() && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
@@ -176,6 +192,9 @@ public class AddressParserServiceImpl implements AddressParserService {
addr.setKeywords(poi);
}
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isNotBlank(addr.getKeywords()) && addr.getOrgAddress().indexOf(addr.getKeywords())>-1){
String str = AddressParserUtil.match(Pattern.compile(".*村(.*(组|队))?\\d+(号)?$"), addr.getOrgAddress().substring(addr.getOrgAddress().indexOf(addr.getKeywords()),addr.getOrgAddress().length()));
if(StringUtils.isNotBlank(str)){
@@ -187,26 +206,80 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setSuggestCity(aliyunBo.getCity());
}
}
}
AddressMatchMetaData meta = null;
if(StringUtils.isBlank(addr.getCity())){
addr.addKeyword(addr.getOrgAddress());
meta = getMapMeta(addr);
if(meta != null){
addr.setProvince(meta.getMapProvince());
addr.setCity(meta.getMapCity());
addr.setDistrict(meta.getDistrict());
}
}
addr.addKeyword(addr.getOrgAddress());
if(params.isUseMap()){
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP);
}else{
matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.CPDB);
}
/*if(jsonObj != null){
//乡镇优化
String township = jsonObj.containsKey("town") ? jsonObj.getString("town") : "";
if(StringUtils.isBlank(addr.getTownship()) && StringUtils.isNotBlank(township)){
String town = AddressParserUtil.match(AddressParserPatterns.TOWN, township);
if(StringUtils.isNotBlank(town)){
addr.setTownship(town);
}
}
//道路优化(两边均去掉乡镇)
String street = addr.getStreet();
String streetNumber = addr.getStreetNumber();
String streetCrf = jsonObj.containsKey("road") ? jsonObj.getString("road") : "";
String streetNumberCrf = jsonObj.containsKey("roadno") ? jsonObj.getString("roadno") : "";
//去掉路名中含有乡镇名的
if(StringUtils.isNotBlank(addr.getTownship())){
street = StringUtils.isNotBlank(street) ? street.replace(addr.getTownship(),"") : street;
streetCrf = StringUtils.isNotBlank(streetCrf) ? streetCrf.replace(addr.getTownship(),"") : streetCrf;
}
//crf解析出的路名为空、与Java解析出路名一致或不满足正则则不处理
if(StringUtils.isNotBlank(streetCrf) && !streetCrf.equals(street) && StringUtils.isNotBlank(AddressParserUtil.match(AddressParserPatterns.STREET, streetCrf))){
//Java解析出路名为空
if(StringUtils.isBlank(street)){
street = streetCrf;
if(StringUtils.isNotBlank(streetNumberCrf)){
streetNumber = streetNumberCrf;
}
}else{
//CRF包含java解析出的路名或Java解析出的路名包含CRF解析出的路名并且多余部分也满足路名正则
if(streetCrf.contains(street) || (street.contains(streetCrf) && StringUtils.isBlank(AddressParserUtil.match(AddressParserPatterns.STREET, street.replace(streetCrf,""))))){
street = streetCrf;
}
}
addr.setStreet(street);
addr.setStreetNumber(streetNumber);
}
//poi处理
//没有匹配上库中小区时
if(StringUtils.isBlank(matchedBuilding.getId())){
String poi = (jsonObj.containsKey("community") ? jsonObj.getString("community") : "") +
(jsonObj.containsKey("poi") ? jsonObj.getString("poi") : "") +
(jsonObj.containsKey("subpoi") ? jsonObj.getString("subpoi") : "");
//去掉路名中含有乡镇名和路名 getParsedPropertyName
if(StringUtils.isNotBlank(addr.getTownship())){
poi = poi.replace(addr.getTownship(),"");
}
if(StringUtils.isNotBlank(addr.getStreet())){
poi = poi.replace(addr.getStreet(),"");
}
addr.setParsedPropertyName(poi);
}
//楼栋、单元、室号
String houseNo = (jsonObj.containsKey("houseno") ? jsonObj.getString("houseno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String cellNo = (jsonObj.containsKey("cellno") ? jsonObj.getString("cellno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String floorNo = (jsonObj.containsKey("floorno") ? jsonObj.getString("floorno") : "").replaceAll("[\u4E00-\u9FA5]", "");
String roomNo = (jsonObj.containsKey("roomno") ? jsonObj.getString("roomno") : "").replaceAll("[\u4E00-\u9FA5]", "");
addr.setOrgBuildingNumber(houseNo);
addr.setBuildingNumber(houseNo);
addr.setOrgUnitNumber(cellNo);
addr.setUnitNumber(cellNo);
addr.setFloor(floorNo);
addr.setOrgRoomNumber(roomNo);
addr.setRoomNumber(roomNo);
}*/
matchedBuilding.setProvince(addr.getProvince());
matchedBuilding.setSearchAddress(addr);
matchedBuilding.setSuggestName(getSuggestName(matchedBuilding));
@@ -215,7 +288,8 @@ public class AddressParserServiceImpl implements AddressParserService {
completBuildingField(matchedBuilding, aliyunBo, params);
}
Address tempAddr = matchedBuilding.getSearchAddress();
if(StringUtils.isBlank(tempAddr.getStreet()) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
if((StringUtils.isBlank(tempAddr.getStreet()) || StringUtils.isBlank(tempAddr.getStreetNumber())) && AddressParserGoal.Match.equals(addr.getAddressParserGoal())){
AddressMatchMetaData meta = getMapMeta(addr);
if(meta != null && (tempAddr.getKeywords()!=null && tempAddr.getKeywords().equals(meta.getMainName()))){
matchedBuilding.setAddress(meta.getAddress());
}
@@ -255,6 +329,25 @@ public class AddressParserServiceImpl implements AddressParserService {
}
return null;
}
/**
* TODO 省市区乡镇采用原始拆分数据,若乡镇数据没有使用
* @param address
* @return
*/
/*private JSONObject getDataFromCRF(String address){
HttpRequest request = new HttpRequest();
Map<String,String> params = new HashMap<>();
params.put("address",address);
try {
JSONObject resp = request.getJsonContent("http://127.0.0.1:8802/standard", params, "UTF-8");
System.out.println(resp);
return resp;
}catch(Exception e){
}
return null;
}*/
/**
* 信息不全时使用阿里接口补全数据
* @param matchedBuilding
@@ -271,33 +364,44 @@ public class AddressParserServiceImpl implements AddressParserService {
}
}
Address addr = matchedBuilding.getSearchAddress();
if(StringUtils.isBlank(addr.getProvince())){
if(StringUtils.isNotBlank(aliyunBo.getProv())){
addr.setProvince(formatProvince(aliyunBo.getProv()));
}
if(StringUtils.isNotBlank(aliyunBo.getCity())){
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isNotBlank(aliyunBo.getDistrict())){
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
}
addr.setTownship(aliyunBo.getTown());
/*if(StringUtils.isBlank(addr.getProvince())){
addr.setProvince(formatProvince(aliyunBo.getProv()));
}
if(StringUtils.isBlank(addr.getCity())){
addr.setCity(RegionData.formatCity(aliyunBo.getCity()));
}
if(StringUtils.isBlank(addr.getDevelopmentZone())){
addr.setDevelopmentZone(aliyunBo.getDevzone());
}
if(StringUtils.isBlank(addr.getDistrict())){
addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict()));
}
if(StringUtils.isBlank(addr.getTownship())){
addr.setTownship(aliyunBo.getTown());
}*/
if(StringUtils.isBlank(addr.getDevelopmentZone())){
addr.setDevelopmentZone(aliyunBo.getDevzone());
}
/*addr.setOrgBuildingNumber(aliyunBo.getHouseno());
addr.setBuildingNumber(aliyunBo.getHouseno());
addr.setOrgUnitNumber(aliyunBo.getCellno());
addr.setUnitNumber(aliyunBo.getCellno());*/
if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(²ã|Â¥)$")){
addr.setRoomNumber(aliyunBo.getRoomno());
/*if(aliyunBo.getRoomno()!=null && !aliyunBo.getRoomno().matches(".*(层|楼)$")){
addr.setOrgRoomNumber(aliyunBo.getRoomno());
addr.setRoomNumber(aliyunBo.getRoomno());
}else{
addr.setRoomNumber(null);
addr.setRoomNumber(null);
}
}*/
/*String aliRoom = aliyunBo.getRoomno() == null ? null : aliyunBo.getRoomno().replaceAll("[\u4E00-\u9FA5]", "");
String orgRoom = addr.getOrgRoomNumber() == null ? null : addr.getOrgRoomNumber().replaceAll("[\u4E00-\u9FA5]", "");
@@ -1037,20 +1141,24 @@ public class AddressParserServiceImpl implements AddressParserService {
if(StringUtils.isNotBlank(address.getTownship())){
formattedAddressBuilder.append(address.getTownship());
}
if(StringUtils.isNotBlank(building.getAddress())){
Address orgAddr = new Address();
orgAddr.setOrgAddress(building.getAddress());
Address addr = AddressProcessors.process(orgAddr);
if(StringUtils.isBlank(address.getStreet())){
address.setStreet(addr.getStreet());
}
if(StringUtils.isBlank(address.getStreetNumber())){
address.setStreetNumber(addr.getStreetNumber());
}
}
if(StringUtils.isNotBlank(address.getStreet())){
formattedAddressBuilder.append(address.getStreet());
}
if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){
formattedAddressBuilder.append(address.getStreetNumber());
}
if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){
Address orgAddr = new Address();
orgAddr.setOrgAddress(building.getAddress());
Address addr = AddressProcessors.process(orgAddr);
building.getSearchAddress().setStreet(addr.getStreet());
building.getSearchAddress().setStreetNumber(addr.getStreetNumber());
formattedAddressBuilder.append((addr.getStreet()==null?"":addr.getStreet())+(addr.getStreetNumber()==null?"":addr.getStreetNumber()));
}
if(StringUtils.isNotBlank(building.getSuggestName())){
formattedAddressBuilder.append(building.getSuggestName());
}
@@ -1077,8 +1185,8 @@ public class AddressParserServiceImpl implements AddressParserService {
formattedAddressBuilder.append(address.getUnitNumber() + "单元");
}
if(StringUtils.isNotBlank(address.getRoomNumber())){
formattedAddressBuilder.append(address.getRoomNumber());
// formattedAddressBuilder.append(address.getRoomNumber() + "ÊÒ");
// formattedAddressBuilder.append(address.getRoomNumber());
formattedAddressBuilder.append(address.getRoomNumber() + "室");
}
address.setFormattedAddress(formattedAddressBuilder.toString());
@@ -77,6 +77,16 @@ public class Address {
private String debugName;
private AliyunBo aliyunBo;
//使用CRF切分的地址
private String crfAddress;
public String getCrfAddress() {
return crfAddress;
}
public void setCrfAddress(String crfAddress) {
this.crfAddress = crfAddress;
}
public AddressParserGoal getAddressParserGoal() {
return addressParserGoal;
@@ -64,9 +64,7 @@ public class Building {
this.alias = d.getAlias();
this.addrAlias = d.getAddrAlias();
this.propertyType = d.getEstateType2();
if(StringUtils.isNotBlank(d.getStreetCommunity())){
this.township = d.getStreetCommunity();
}
this.township = d.getTownship();
if(d.getLat() != null && d.getLng() != null){
this.lat = d.getLat()/10000000000d;
this.lng = d.getLng()/10000000000d;
@@ -90,9 +88,7 @@ public class Building {
this.lat = d.getLat2()/10000000000d;
this.lng = d.getLng2()/10000000000d;
}
if(StringUtils.isNotBlank(d.getStreetCommunity())){
this.township = d.getStreetCommunity();
}
this.township = d.getTownship();
}
public Building() {
@@ -25,7 +25,15 @@ public class MatchParams {
private boolean useMap;
/**是否使用阿里云接口*/
private boolean useAliyunApi;
//是否使用CRF算法
private boolean useCrf;
public boolean isUseCrf() {
return useCrf;
}
public void setUseCrf(boolean useCrf) {
this.useCrf = useCrf;
}
public boolean isContainsLocation() {
return containsLocation;
}
@@ -14,6 +14,7 @@ public class StageAndPartitionProcessor extends AbstractAddressProcessor {
@Override
public RepeatStatus process(Address address) {
String orgAddr = address.getEffectiveAddress();
address.setCrfAddress(address.getAddress());
MatchResult matchResult = matchFirst(AddressParserPatterns.STAGE_PATTERN, address.getEffectiveAddress());
if (matchResult != null) {
String formatStage = CNNMFilter.convertArabicToCnNumeric(matchResult.matchContent);
@@ -1,5 +1,7 @@
package com.fangjia.service.api;
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeRequest;
import com.aliyuncs.address_purification.model.v20191118.GetAddressDivisionCodeResponse;
import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest;
import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse;
import com.fangjia.core.util.JsonUtils;
@@ -13,6 +15,9 @@ import net.sf.json.JSONObject;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import java.util.ArrayList;
import java.util.List;
public class AliyunServiceImpl implements AliyunService {
protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass());
@@ -35,20 +40,44 @@ public class AliyunServiceImpl implements AliyunService {
if(StringUtils.isNotBlank(city)){
request.setDefaultCity(city);
}
AliyunBo aliyunBo = new AliyunBo();
try{
StructureAddressResponse response = client.getAcsResponse(request);
JSONObject responseJson = JSONObject.fromObject(response.getData());
String value = (String) responseJson.get("structure");
AliyunBo aliyunBo = new AliyunBo();
for (String key:value.split("\t")){
String[] arry = key.split("=");
wrapField(arry[0],arry[1], aliyunBo);
}
return aliyunBo;
}catch(Exception e){
LOGGER.error("", e);
return null;
LOGGER.error("调用阿里标准化接口失败:", e);
}
try{
GetAddressDivisionCodeRequest divisionRequest = new GetAddressDivisionCodeRequest();
divisionRequest.setServiceCode("addrp");
divisionRequest.setAppKey(APPKEY);
divisionRequest.setText(address);
GetAddressDivisionCodeResponse divisionResponse = client.getAcsResponse(divisionRequest);
JSONObject responseJson = JSONObject.fromObject(divisionResponse.getData());
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<String>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
aliyunBo.setProv(array[0]);
aliyunBo.setCity(array[1]);
aliyunBo.setDistrict(array[2]);
aliyunBo.setTown(array[3]);
}catch(Exception e){
LOGGER.error("调用阿里行政区划识别接口失败", e);
}
return aliyunBo;
}
private void wrapField(String field,String value, AliyunBo aliyunBo){
@@ -81,26 +110,59 @@ public class AliyunServiceImpl implements AliyunService {
}
}
public static void main(String[] args) throws ClientException {
DefaultProfile profile = DefaultProfile.getProfile(
String value = "";
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<String>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
System.out.println(array[0]);
System.out.println(array[1]);
System.out.println(array[2]);
System.out.println(array[3]);
/*DefaultProfile profile = DefaultProfile.getProfile(
REGIONID,
ACCESSKEYID,
SECRET);
IAcsClient client = new DefaultAcsClient(profile);
StructureAddressRequest request = new StructureAddressRequest();
// StructureAddressRequest request = new StructureAddressRequest();
GetAddressDivisionCodeRequest request = new GetAddressDivisionCodeRequest();
request.setServiceCode("addrp");
request.setAppKey(APPKEY);
request.setText("杭州市文一西路969号");
request.setDefaultProvince("浙江省");
request.setDefaultCity("杭州市");
request.setDefaultDistrict("余杭区");
StructureAddressResponse response = client.getAcsResponse(request);
request.setText("天青路260号");
// request.setDefaultProvince("浙江省");
// request.setDefaultCity("杭州市");
// request.setDefaultDistrict("余杭区");
// StructureAddressResponse response = client.getAcsResponse(request);
GetAddressDivisionCodeResponse response = client.getAcsResponse(request);
JSONObject responseJson = JSONObject.fromObject(response.getData());
String value = (String) responseJson.get("structure");
JSONObject json = new JSONObject();
System.out.println(responseJson);
String value = (String) ((JSONObject) responseJson.get("division_info")).get("division_name");
String[] array = value.replace("\t", ";").split(";");
if(array.length > 4){
List<String> tmp = new ArrayList<String>();
for(String str:array){
if(str!=null && str.length()!=0){
tmp.add(str);
}
}
array= tmp.toArray(new String[4]);
}
System.out.println(array[0]);
System.out.println(array[1]);
System.out.println(array[2]);
System.out.println(array[3]);*/
/*JSONObject json = new JSONObject();
for (String key:value.split("\t")){
String[] arry = key.split("=");
json.put(arry[0],arry[1]);
}
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));
System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class));*/
}
}
@@ -5,18 +5,7 @@ import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URLEncoder;
import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.*;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
@@ -5080,7 +5069,7 @@ public class AddressServiceImpl implements AddressService {
double similar = AddressParserUtil.getSimilarityRatio(formatAddress1_pinyin, formatAddress2_pinyin);
String level = getAddressSimilarityLevel(json1, json2);
resultMap.put("isSameAddress", sameAddress);
resultMap.put("score", similar);
// resultMap.put("score", similar);
resultMap.put("level", level);
resultMap.put("formattedAddress_1", getFromJson(json1,"formattedAddress"));
resultMap.put("formattedAddress_2", getFromJson(json2,"formattedAddress"));
@@ -5138,30 +5127,51 @@ public class AddressServiceImpl implements AddressService {
return null;
}
/**
* 获取两地址的相似度评级(最小颗粒度)
* 获取两地址的相似度评级(相同数据的最小颗粒度)
* @return
*/
private String getAddressSimilarityLevel(JSONObject json1, JSONObject json2){
if(StringUtils.isNotBlank(getFromJson(json1,"roomNumber")) || StringUtils.isNotBlank(getFromJson(json2,"roomNumber"))){
return "roomNumber";
}else if(StringUtils.isNotBlank(getFromJson(json1,"unitNumber")) || StringUtils.isNotBlank(getFromJson(json2,"unitNumber"))){
return "unitNumber";
}else if(StringUtils.isNotBlank(getFromJson(json1,"buildingNumber")) || StringUtils.isNotBlank(getFromJson(json2,"buildingNumber"))){
return "buildingNumber";
}else if(StringUtils.isNotBlank(getFromJson(json1,"name")) || StringUtils.isNotBlank(getFromJson(json2,"name"))){
return "name";
}else if(StringUtils.isNotBlank(getFromJson(json1,"street")) || StringUtils.isNotBlank(getFromJson(json2,"street"))){
return "street";
}else if(StringUtils.isNotBlank(getFromJson(json1,"township")) || StringUtils.isNotBlank(getFromJson(json2,"township"))){
return "township";
}else if(StringUtils.isNotBlank(getFromJson(json1,"district")) || StringUtils.isNotBlank(getFromJson(json2,"district"))){
return "district";
}else if(StringUtils.isNotBlank(getFromJson(json1,"city")) || StringUtils.isNotBlank(getFromJson(json2,"city"))){
return "city";
}else if(StringUtils.isNotBlank(getFromJson(json1,"province")) || StringUtils.isNotBlank(getFromJson(json2,"province"))){
return "province";
String level = null;
LinkedList<String> list = new LinkedList<String>(Arrays.asList("province", "city", "district", "township", "street", "streetNumber"
, "name", "buildingNumber", "unitNumber", "roomNumber"));
for(String key:list){
if(continueJudgeValue(json1, json2, key) == 3){
return level;
}else{
level = continueJudgeValue(json1, json2, key) == 1 ? key : (level == null ? null : level);
}
}
return null;
return level;
}
/**
* 判断level时是否继续往下判断
* @param json1
* @param json2
* @param key
* @return 1 继续判断并且level有值 2继续判断level没值 3 停止
*/
private Integer continueJudgeValue(JSONObject json1, JSONObject json2, String key){
if("buildingNumber".equals(key) || "unitNumber".equals(key) || "roomNumber".equals(key)){
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))
&& getFromJson(json1,key).equals(getFromJson(json2,key))){
return 1;
}else{
return 3;
}
}
if(StringUtils.isNotBlank(getFromJson(json1,key)) && StringUtils.isNotBlank(getFromJson(json2,key))){
if("name".equals(key)){
return verifySamePoi(getFromJson(json1,key),getFromJson(json2,key)) ? 1 : 3;
}else{
if(getFromJson(json1,key).equals(getFromJson(json2,key))){
return 1;
}else{
return 3;
}
}
}
return 2;
}
/**
* 判断省、市、区、乡镇、楼栋、单元、室号是否相同(两者全为空时返回true)
@@ -5210,6 +5220,10 @@ public class AddressServiceImpl implements AddressService {
if(!verifySameField(name1_stage, name2_stage, true) || !verifySameField(name1_partition, name2_partition, true)|| !verifySameField(name1_number, name2_number, true)){
return false;
}
return isSameName(name1, name2);
}
private boolean isSameName(String name1, String name2){
String name_1_pinyin = AddressParserUtil.toHanYuPinYin(name1);
String name_2_pinyin = AddressParserUtil.toHanYuPinYin(name2);
//拼音或文本数据包含并且
@@ -5222,7 +5236,6 @@ public class AddressServiceImpl implements AddressService {
}
return false;
}
/**
* 设置区域消费指数
* @param checkTr