diff --git a/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/GuzhiController.java b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/GuzhiController.java new file mode 100644 index 000000000..c3d1c6116 --- /dev/null +++ b/projects/fangjia-coresite/src/main/java/com/fangjia/coresite/controllers/GuzhiController.java @@ -0,0 +1,10 @@ +package com.fangjia.coresite.controllers; + +import com.fangjia.coresite.model.FangjiaModel; + +public class GuzhiController { + + public FangjiaModel mapAction(FangjiaModel model){ + return model; + } +} diff --git a/projects/fangjia-coresite/src/main/webapp/company/about.ftl b/projects/fangjia-coresite/src/main/webapp/company/about.ftl index 0f386bf92..b1af2a8cc 100644 --- a/projects/fangjia-coresite/src/main/webapp/company/about.ftl +++ b/projects/fangjia-coresite/src/main/webapp/company/about.ftl @@ -124,7 +124,7 @@ - +<#-- --> diff --git a/projects/fangjia-coresite/src/main/webapp/guzhi/map.ftl b/projects/fangjia-coresite/src/main/webapp/guzhi/map.ftl new file mode 100644 index 000000000..4b013115e --- /dev/null +++ b/projects/fangjia-coresite/src/main/webapp/guzhi/map.ftl @@ -0,0 +1,72 @@ + + + + + + + 基本热力图 + + + +
+ + + + + + \ No newline at end of file diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index b26aed431..acf9c9b6f 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1691,7 +1691,14 @@ public class AddressController { } String result = addressMatchProxyService.standard(convertModelToMap(model)); if (result != null) { - mav.setModel(result); + JSONObject json = JSONObject.fromObject(result); + int code = json.getInt("code"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); } else { mav.setModel(BasicModel.buildServerErrorModel().setMsg("·þÎñÉý¼¶ÖÐ...")); } @@ -1730,16 +1737,20 @@ public class AddressController { mav.setModel(BasicModel.buildBadRequestModel("µØÖ·²»ÄÜΪ¿Õ")); return mav; } - if (StringUtils.isBlank(model.getCity())) { + /*if (StringUtils.isBlank(model.getCity())) { mav.setModel(BasicModel.buildBadRequestModel("³ÇÊв»ÄÜΪ¿Õ")); return mav; - } + }*/ MatchParams matchParams = new MatchParams(); matchParams.setCity(model.getCity()); matchParams.setAddress(model.getAddress()); matchParams.setAddressParserGoal(AddressParserGoal.Match); matchParams.setPreciseLevel(PreciseLevel.STRICT); matchParams.setProvince(model.getProvince()); + +// matchParams.setUseMap(true); + matchParams.setUseAliyunApi(true); + matchParams.setContainsLocation(model.isContainsLocation()); Building building = addressParserService.standard(matchParams); Address address = building.getSearchAddress(); UniversalDto dto = UniversalDto.newDto(); @@ -1755,10 +1766,36 @@ public class AddressController { dto.append("street", address.getStreet()); dto.append("streetNumber", address.getStreetNumber()); dto.append("name", building.getSuggestName()); + if(StringUtils.isBlank(building.getSuggestName())){ + if(StringUtils.isNotBlank(address.getParsedPropertyName())){ + dto.append("name", address.getParsedPropertyName()); + }else if(StringUtils.isNotBlank(address.getOrgKeywords())){ + dto.append("name", address.getOrgKeywords()); + } + } + if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ + String addr = building.getAddress(); + String street = AddressParserUtil.extractStreet(addr); + if(street != null){ + dto.append("street", street); + if(addr.length() != street.length()){ + dto.append("streetNumber", AddressParserUtil.match(AddressParserUtil.STREET_NUMBER_CONNECTOR_PATTERN, addr)); + } + }else if(StringUtils.isNotBlank(building.getStreet())){ + dto.append("street", building.getStreet()); + if(StringUtils.isNotBlank(building.getStreetno())){ + dto.append("streetNumber", building.getStreetno()); + } + } + } dto.append("buildingNumber", address.getBuildingNumber()); dto.append("unitNumber", address.getUnitNumber()); dto.append("roomNumber", address.getRoomNumber()); dto.append("formattedAddress", address.getFormattedAddress()); + if(matchParams.isContainsLocation()){ + dto.append("lat", building.getLat()); + dto.append("lng", building.getLng()); + } mav.setModel(BasicModel.buildOKModel().setResult(dto)); return mav; } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java index 92345873e..3d362bbb9 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddesssMatchModel.java @@ -19,6 +19,16 @@ public class AddesssMatchModel extends FangjiaApiModel { private String province; private String debugName; private boolean useMap; + + public boolean isContainsLocation() { + return containsLocation; + } + + public void setContainsLocation(boolean containsLocation) { + this.containsLocation = containsLocation; + } + + private boolean containsLocation; private boolean debug; public String getCity() { return city; diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java index adf25301a..8c41c06ab 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/model/AddressModel.java @@ -36,7 +36,15 @@ public class AddressModel extends FangjiaApiModel { private Integer pageNo; // ÊÇ·ñÓÃÓÚ¼ì²é private String dataLevel; + public boolean isContainsLocation() { + return containsLocation; + } + public void setContainsLocation(boolean containsLocation) { + this.containsLocation = containsLocation; + } + + private boolean containsLocation; public boolean isIncludeMaps() { return includeMaps; } diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 2ac4bbb76..cd0782adc 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -111,6 +111,11 @@ fangjia-datastore-driver 3.9.1 + + cc.blynk.clickhouse + clickhouse4j + 1.4.4 + redis.clients jedis diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 5ed93cde1..7dae269d1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -83,11 +83,11 @@ class HouseExport { // exportRentCSV(city, districts, 20200201, 20210301, 2020); }else{ - exportRentCSV(city, districts, 20170801, 20181001, 2017) - exportRentCSV(city, districts, 20200201, 20210301, 2020); +// exportRentCSV(city, districts, 20170801, 20181001, 2017) +// exportRentCSV(city, districts, 20200201, 20210301, 2020); exportSellCSV(city, districts, 20170801, 20181001, 2017); - exportSellCSV(city, districts, 20200201, 20210301, 2020); +// exportSellCSV(city, districts, 20200201, 20210301, 2020); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy new file mode 100644 index 000000000..5eed415f1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy @@ -0,0 +1,82 @@ +package com.fangjia.script.wenyan.temp + +import cc.blynk.clickhouse.copy.CopyManager +import cc.blynk.clickhouse.copy.CopyManagerFactory +import com.fangjia.core.util.DateUtil +import net.sf.json.JSONNull +import net.sf.json.JSONObject +import org.bson.types.ObjectId +import ru.yandex.clickhouse.ClickHouseConnection +import ru.yandex.clickhouse.ClickHouseDriver +import ru.yandex.clickhouse.settings.ClickHouseProperties + +import java.sql.PreparedStatement +import java.sql.Timestamp + +//ClickHouseDriver clickHouseDriver = new ClickHouseDriver() +//ClickHouseProperties clickHouseProperties = new ClickHouseProperties() +//clickHouseProperties.connectionTimeout = 1000 * 3600 * 24 +//clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24 +//clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24 +//clickHouseProperties.user = "default" +//clickHouseProperties.password = "goojia123456" +//ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties) +//CopyManager copyManager = CopyManagerFactory.create(connection) +//copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv"); + + +//PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") +//def cols = ["id", "city", "region", "name", "district_name", "address", "room", "hall", "toilet", "area", "avg_price", "total_price ", "floor_info", "height", "floor", "direction", "fitment", "p_date", "publish_date", "c_date", "m_date"] +//int count = 0 +//new File("D:\\½¨ÐвâÊÔÊý¾Ý").eachFile { def file -> +// file.eachLine("UTF-8", { def line -> +// String id = line.substring(0, line.indexOf("\t")) +// String jsonStr = line.substring(line.indexOf("\t") + 1) +// count++ +// JSONObject json = JSONObject.fromObject(jsonStr) +// json.put("id", id) +// for (int i = 0; i < cols.size(); i++) { +// if (cols[i].toString().endsWith("date") && "p_date" != cols[i]) { +// if ("c_date" == cols[i]) { +// if (json.get("m_date") instanceof Long) { +// sth.setTimestamp(i + 1, new Timestamp(json.get("m_date"))) +// } else { +// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get("m_date"))).getTime())) +// +// } +// } else { +// if (json.get(cols[i]) instanceof String) { +// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get(cols[i]))).getTime())) +// } else { +// sth.setTimestamp(i + 1, new Timestamp(json.get(cols[i]))) +// +// } +// } +// } else { +// if (json.get(cols[i]) instanceof JSONNull) { +// sth.setObject(i + 1, null) +// } else { +// sth.setObject(i + 1, json.get(cols[i])) +// } +// } +// } +// sth.addBatch() +// if (count % 30000 == 0) { +// sth.executeBatch() +// println(new Date().format("HH:mm:ss " + count)) +// } +// }) +//} +//sth.execute() +def ids = new HashSet() +for(int i =0 ; i < 10000 * 10000; i++){ + if(i%10000 == 0){ + println(i) + } + def id = new ObjectId().toString() + if(ids.contains(id)){ + println(id) + }else{ + ids << id + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy index d314114dc..b245cfec6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Price.groovy @@ -1,17 +1,16 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.ProvinceData -import com.fangjia.core.RegionData -import com.fangjia.script.util.CommonUtils + import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject -ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\½¨ÐÐÉÊý¾Ýͳ¼Æ.xlsx",{List item-> - String city = RegionData.formatCity(item["A"]) - if(city){ - item["G"] = DbUtils.seaweed.count(DbUtils.buildSeaweedQuery(RegionData.formatCity(item["A"]))) - }else{ - item["G"] = null +def data = [["³ÇÊÐ", "ÇøÓò", "Ð¡ÇøÃû", "±ðÃû", "µØÖ·", "µØÖ·±ðÃû"]] +["ID", "±±¾©", "ÖØÇì", "¹þ¶û±õ"].each {def city-> + DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district-> + def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")] + println(item) + data << item } - -}) \ No newline at end of file +} +ExcelUtils.write("D:\\´ý×¥È¡Ð¡Çø.xlsx", data, "Êý¾Ý") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy new file mode 100644 index 000000000..c75c8611b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -0,0 +1,42 @@ +package com.fangjia.script.wenyan.temp + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20210201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210301", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [["Óû§ID", "Óû§Ãû³Æ", "IP", "·ÃÎÊÁ´½Ó", "·ÃÎÊʱ¼ä", "dataStatus", "code", "msg", "²ÎÊý", "resMap"]] +while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + count++ + OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) + if(!result.buildYear || !result.avgPrice){ + return + } + def params = JSONObject.fromObject(doc.parameter_map) + def key = "${params.address}_${params.size}".toString() + if (!hashSet.contains(key)) { + def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item + hashSet << key + } else { + println("ÒÑ´æÔÚ") + } + } + startDate = nextDate +} +ExcelUtils.write("D:\\ËÄ´¨½õ³Ç${startDate.format('yyyyMM')}.xlsx", data, "²éѯÈÕÖ¾") \ No newline at end of file diff --git a/projects/fangjia-service/pom.xml b/projects/fangjia-service/pom.xml index 95f09c4f9..723fc818c 100644 --- a/projects/fangjia-service/pom.xml +++ b/projects/fangjia-service/pom.xml @@ -214,5 +214,15 @@ jieba-analysis 1.0.2 + + com.aliyun + aliyun-java-sdk-address-purification + 1.0.0 + + + com.aliyun + aliyun-java-sdk-core + 4.5.19 + diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java index f9205e4b0..d323f797b 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserServiceImpl.java @@ -3,6 +3,7 @@ package com.fangjia.service.address; import com.fangjia.core.RegionData; import com.fangjia.core.util.DataUtil; import com.fangjia.core.util.GeoUtils; +import com.fangjia.core.util.StringUtil; import com.fangjia.dal.QueryParam; import com.fangjia.dal.common.DistrictKey; import com.fangjia.dal.common.ServerAreaData; @@ -14,6 +15,8 @@ import com.fangjia.dal.seaweed.model.Seaweed; import com.fangjia.service.address.mode.*; import com.fangjia.service.address.processor.AddressDistrictProcessors; import com.fangjia.service.address.processor.AddressProcessors; +import com.fangjia.service.api.AliyunService; +import com.fangjia.service.bo.AliyunBo; import com.fangjia.service.datastore.DistrictService; import com.fangjia.service.mongo.addrepo.AddressRepoService; import com.fangjia.service.mongo.bkoff.HouseLoudongService; @@ -41,6 +44,7 @@ public class AddressParserServiceImpl implements AddressParserService { private HouseLoudongService houseLoudongService; private LoudongService loudongService; private SeaweedDao seaweedDao; + private AliyunService aliyunService; private final String[] MATCH_PROPERTY_FIELDS = new String[]{"id", "city", "region", "name", "address", "category", "block", "alias", "addrAlias", "estateType2", "lat2", "lng2", "streetCommunity"}; private static final Logger logger = Logger.getLogger(AddressParserServiceImpl.class); @@ -159,6 +163,18 @@ public class AddressParserServiceImpl implements AddressParserService { return building; } Building matchedBuilding; + //keywordsʹÓÃaliyun½Ó¿Ú·µ»ØµÄpoi + AliyunBo aliyunBo = null; + if(params.isUseAliyunApi()){ + aliyunBo = aliyunService.addressStand(params.getAddress(), params.getCity()); + if(aliyunBo != null){ + String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", ""); + addr.setKeywords(poi); + if(StringUtils.isNotBlank(aliyunBo.getCity())){ + addr.setSuggestCity(aliyunBo.getCity()); + } + } + } if(params.isUseMap()){ matchedBuilding = findMatchedBuilding(addr, params.getBuildingType(), MetaDataType.MAP); }else{ @@ -168,9 +184,75 @@ public class AddressParserServiceImpl implements AddressParserService { matchedBuilding.setSearchAddress(addr); matchedBuilding.setSuggestName(getSuggestName(matchedBuilding)); matchedBuilding.setSuggestAddress(getSuggestAddress(matchedBuilding)); + if(params.isUseAliyunApi() && aliyunBo != null){ + completBuildingField(matchedBuilding, aliyunBo, params); + } return matchedBuilding; } + /** + * ÐÅÏ¢²»È«Ê±Ê¹Óð¢Àï½Ó¿Ú²¹È«Êý¾Ý + * @param matchedBuilding + * @param aliyunBo + * @return + */ + void completBuildingField(Building matchedBuilding, AliyunBo aliyunBo, MatchParams params){ + String poi = ((aliyunBo.getPoi() == null ? "":aliyunBo.getPoi()) + (aliyunBo.getSubpoi() == null ? "":aliyunBo.getSubpoi())).replaceAll("\\s*", ""); + if(StringUtils.isBlank(matchedBuilding.getSuggestName())){ + matchedBuilding.setSuggestName(poi); + }else if(matchedBuilding.getSuggestName()!=null && StringUtils.isNotBlank(poi) && params.getAddress().contains(poi)){ + if(!params.getAddress().contains(matchedBuilding.getSuggestName()) || poi.contains(matchedBuilding.getSuggestName())){ + matchedBuilding.setSuggestName(poi); + } + } + Address addr = matchedBuilding.getSearchAddress(); + if(StringUtils.isBlank(addr.getProvince())){ + addr.setProvince(formatProvince(aliyunBo.getProv())); + } + if(StringUtils.isBlank(addr.getCity())){ + addr.setCity(RegionData.formatCity(aliyunBo.getCity())); + } + + if(StringUtils.isBlank(addr.getDevelopmentZone())){ + addr.setDevelopmentZone(aliyunBo.getDevzone()); + } + if(StringUtils.isBlank(addr.getDistrict())){ + addr.setDistrict(RegionData.formatRegion(aliyunBo.getDistrict())); + } + if(StringUtils.isBlank(addr.getTownship())){ + addr.setTownship(aliyunBo.getTown()); + } + if(StringUtils.isBlank(addr.getBuildingNumber())){ + addr.setBuildingNumber(aliyunBo.getHouseno()); + } + if(StringUtils.isBlank(addr.getUnitNumber())){ + addr.setUnitNumber(aliyunBo.getCellno()); + } + if(StringUtils.isBlank(addr.getRoomNumber())){ + addr.setRoomNumber(aliyunBo.getRoomno()); + } + if(StringUtils.isBlank(addr.getStreet())){ + addr.setStreet(aliyunBo.getRoad()); + } + if(StringUtils.isBlank(addr.getStreetNumber())){ + addr.setStreetNumber(aliyunBo.getRoadno()); + } + matchedBuilding.setSearchAddress(addr); + } + + String formatProvince(String province){ + if(StringUtils.isBlank(province)){ + return province; + } + Map> provinceData = ProvinceData.getAllProvince(); + for (String name : provinceData.keySet()) { + if(province.contains(name)){ + return name; + } + } + return null; + } + Building findMatchedBuilding(Address addr,BuildingType buildingType, MetaDataType metaDataType){ AddressScoreResult scoreResult = findMatchedMetaData(addr, buildingType, metaDataType); @@ -830,11 +912,7 @@ public class AddressParserServiceImpl implements AddressParserService { } } String cityDisplay = CityData.getCityDisplay(address.getCity()); - if(cityDisplay != null){ - formattedAddressBuilder.append(cityDisplay); - }else{ - formattedAddressBuilder.append(address.getCity() + "ÊÐ"); - } + formattedAddressBuilder.append(cityDisplay != null ? cityDisplay : (StringUtils.isNotBlank(address.getCity())?(address.getCity()+ "ÊÐ"):"")); if(StringUtils.isNotBlank(address.getDistrict())){ String orgDistrict = RegionData.getOrgName(address.getCity(), address.getDistrict()); if(StringUtils.isNotBlank(orgDistrict) && formattedAddressBuilder.indexOf(orgDistrict) == -1){ @@ -855,17 +933,20 @@ public class AddressParserServiceImpl implements AddressParserService { if(StringUtils.isNotBlank(address.getStreet())){ formattedAddressBuilder.append(address.getStreet()); } - if("ÉϺ£".equals(address.getCity()) && StringUtils.isNotBlank(address.getStreet()) && !address.getStreet().endsWith("Ū")){ - if(StringUtils.isNotBlank(address.getStreetNumber())){ - formattedAddressBuilder.append(address.getStreetNumber()); - } + if(StringUtils.isNotBlank(address.getStreet()) && StringUtils.isNotBlank(address.getStreetNumber())){ + formattedAddressBuilder.append(address.getStreetNumber()); } - if(StringUtils.isNotBlank(building.getSuggestName()) && building.getSuggestName().equals(address.getOrgKeywords())){ + if(StringUtils.isBlank(address.getStreet()) && StringUtils.isNotBlank(building.getAddress())){ + formattedAddressBuilder.append(building.getAddress()); + } + if(StringUtils.isNotBlank(building.getSuggestName())){ formattedAddressBuilder.append(building.getSuggestName()); } - if("ÉϺ£".equals(address.getCity()) && StringUtils.isNotBlank(address.getStreet()) && address.getStreet().endsWith("Ū")){ - if(StringUtils.isNotBlank(address.getStreetNumber())){ - formattedAddressBuilder.append(address.getStreetNumber()); + if(StringUtils.isBlank(building.getSuggestName())){ + if(StringUtils.isNotBlank(address.getParsedPropertyName())){ + formattedAddressBuilder.append(address.getParsedPropertyName()); + }else if(StringUtils.isNotBlank(address.getOrgKeywords())){ + formattedAddressBuilder.append(address.getOrgKeywords()); } } if(StringUtils.isNotBlank(address.getOrgBuildingNumber())){ @@ -987,4 +1068,8 @@ public class AddressParserServiceImpl implements AddressParserService { public void setSeaweedDao(SeaweedDao seaweedDao) { this.seaweedDao = seaweedDao; } + + public void setAliyunService(AliyunService aliyunService) { + this.aliyunService = aliyunService; + } } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserUtil.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserUtil.java index dbc230c2d..e1f4b1338 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserUtil.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/AddressParserUtil.java @@ -21,7 +21,7 @@ import java.util.regex.Pattern; */ public class AddressParserUtil { - static final Pattern STREET_PATTERN = Pattern.compile("^.+(·|½ÖµÀ|Ū|½Ö)"); + static final Pattern STREET_PATTERN = Pattern.compile("^.+(·|½ÖµÀ|Ū|½Ö|µÀ|Ïï)"); static final Pattern CONNECTOR_PATTERN = Pattern.compile("~|-|¡«"); public static final Pattern STREET_NUMBER_CONNECTOR_PATTERN = Pattern.compile("\\d+(~|-|¡«)\\d+ºÅ"); static final Pattern SEPARATOR_PATTERN = Pattern.compile("¡¢|,"); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java index 5968e0e5a..21a79a722 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Address.java @@ -1,6 +1,7 @@ package com.fangjia.service.address.mode; import com.fangjia.service.address.AddressParserUtil; +import com.fangjia.service.bo.AliyunBo; import com.fangjia.service.cnAnalyzer.Token; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.lang.StringUtils; @@ -56,6 +57,7 @@ public class Address { private String effectiveAddressPinYin; private String offset; private String miniAddress; + private String suggestCity; private String suggestDistrict; private String suggestName; private String suggestNamePinYin; @@ -74,6 +76,15 @@ public class Address { /**µ÷ÊÔʱ¶ÏµãÐ¡ÇøÃû*/ private String debugName; + private AliyunBo aliyunBo; + + public AliyunBo getAliyunBo() { + return aliyunBo; + } + + public void setAliyunBo(AliyunBo aliyunBo) { + this.aliyunBo = aliyunBo; + } /** @@ -292,6 +303,14 @@ public class Address { this.floor = floor; } + public String getSuggestCity() { + return suggestCity; + } + + public void setSuggestCity(String suggestCity) { + this.suggestCity = suggestCity; + } + public String getLoopLine() { return loopLine; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java index 1539f8e1c..a217fa8fd 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/Building.java @@ -32,6 +32,25 @@ public class Building { private String suggestAddress; private String map; + public String getStreet() { + return street; + } + + public void setStreet(String street) { + this.street = street; + } + + public String getStreetno() { + return streetno; + } + + public void setStreetno(String streetno) { + this.streetno = streetno; + } + + private String street; + private String streetno; + public Building(District d){ this.city = d.getCity(); this.district = d.getRegion(); diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java index 3ce996636..498b12052 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/mode/MatchParams.java @@ -23,8 +23,23 @@ public class MatchParams { private PreciseLevel preciseLevel; /**ÊÇ·ñʹÓõØÍ¼¿â*/ private boolean useMap; + /**ÊÇ·ñʹÓð¢ÀïÔÆ½Ó¿Ú*/ + private boolean useAliyunApi; + + public boolean isContainsLocation() { + return containsLocation; + } + + public void setContainsLocation(boolean containsLocation) { + this.containsLocation = containsLocation; + } + + /**ÊÇ·ñÊä³ö¾­Î³¶È*/ + private boolean containsLocation; /**µØÖ·½âÎöÄ¿µÄ*/ private AddressParserGoal addressParserGoal; + + public String getProvince() { return province; } @@ -81,6 +96,14 @@ public class MatchParams { this.useMap = useMap; } + public boolean isUseAliyunApi() { + return useAliyunApi; + } + + public void setUseAliyunApi(boolean useAliyunApi) { + this.useAliyunApi = useAliyunApi; + } + public AddressParserGoal getAddressParserGoal() { return addressParserGoal; } diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java index 13980854f..f8ebb93bf 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/CityProcessor.java @@ -121,6 +121,9 @@ public class CityProcessor extends AbstractAddressProcessor { } } } + if(StringUtils.isBlank(address.getCity())){ + address.setCity(address.getSuggestCity()); + } return RepeatStatus.NEXT; } @@ -163,7 +166,14 @@ public class CityProcessor extends AbstractAddressProcessor { }else{ allCity = RegionData.getAllCity(); } + String street = getStreet(address); for(String city : allCity){ + if(street != null){ + MatchResult matchResult = matchFirst(Pattern.compile(city + "[¶«ÄÏÎ÷±±ÖÐÒ»¶þÈýËÄÎåÁùÆß°Ë¾ÅÊ®°Ùǧ\\d]?(·|½Ö|´óµÀ|֧·|´å)"), street); + if(matchResult !=null){ + address = address.replace(matchResult.matchContent, ""); + } + } int index = 0; FragmentWeight weight = weightMap.get(city); if(weight == null){ @@ -221,7 +231,7 @@ public class CityProcessor extends AbstractAddressProcessor { } List weights = new ArrayList(weightMap.values()); if(weights.size() > 0){ - String street = getStreet(address); + /*String street = getStreet(address); if(street != null){ Iterator iterator = weights.iterator(); while(iterator.hasNext()){ @@ -230,7 +240,7 @@ public class CityProcessor extends AbstractAddressProcessor { iterator.remove(); } } - } + }*/ Collections.sort(weights); } if(weights.size() > 0){ diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java index d55b7ae19..5cfba96b5 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/address/processor/DistrictProcessor.java @@ -25,7 +25,7 @@ public class DistrictProcessor extends AbstractAddressProcessor { static final String[] DISTRICT_NOISE = new String[]{"гÇÇø","ÐÂÇø" , "³ÇÇø", "ÊÐÇø" ,"Ï½Çø", "ÆäËûµØÇø", "ÊÐÏ½Çø"}; static final String[] SPECIAL_CITY_DISTRICT_NOISE = new String[]{"³ÇÄÏ", "³Ç±±", "³ÇÎ÷", "³Ç¶«", "гÇÇø","ÐÂÇø" , "³ÇÇø", "ÊÐÇø" ,"Ï½Çø", "ÆäËûµØÇø", "ÊÐÏ½Çø"}; /**¿ª·¢ÇøÁбí*/ - static final String[] DEVELOPMENT_ZONE_LIST = new String[]{"¸ßм¼Êõ¿ª·¢Çø","¾­¼Ã¿ª·¢Çø", "¾­¼Ã¼¼Êõ¿ª·¢Çø", "¾­¿ªÇø" , "¸ßÐÂÇø", "¸ßм¼Êõ¿ª·¢Çø", "¿ª·¢Çø", "¾­¼Ã¼¼Êõ¿ª·¢Çø"}; + static final String[] DEVELOPMENT_ZONE_LIST = new String[]{"¸ßм¼Êõ¿ª·¢Çø","¾­¼Ã¿ª·¢Çø", "¾­¼Ã¼¼Êõ¿ª·¢Çø", "¾­¿ªÇø" , "¸ßÐÂÇø", "¸ßм¼Êõ¿ª·¢Çø", "¿ª·¢Çø", "¾­¼Ã¼¼Êõ¿ª·¢Çø", "¹¤ÒµÔ°Çø"}; /*ÇøÓòÌØÕ÷ÕýÔò*/ static final String DISTRICT_SUFFIX_REGEX = "(ÏØ³Ç|Çø|ÏØ|ÊÐ|Õò)"; diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunService.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunService.java new file mode 100644 index 000000000..91ab454a7 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunService.java @@ -0,0 +1,7 @@ +package com.fangjia.service.api; + +import com.fangjia.service.bo.AliyunBo; + +public interface AliyunService { + AliyunBo addressStand(String address, String city); +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java new file mode 100644 index 000000000..b6b22f5a1 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/api/AliyunServiceImpl.java @@ -0,0 +1,106 @@ +package com.fangjia.service.api; + +import com.aliyuncs.address_purification.model.v20191118.StructureAddressRequest; +import com.aliyuncs.address_purification.model.v20191118.StructureAddressResponse; +import com.fangjia.core.util.JsonUtils; +import com.fangjia.core.util.LoggerFactory; +import com.fangjia.service.bo.AliyunBo; +import com.aliyuncs.DefaultAcsClient; +import com.aliyuncs.IAcsClient; +import com.aliyuncs.exceptions.ClientException; +import com.aliyuncs.profile.DefaultProfile; +import net.sf.json.JSONObject; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; + +public class AliyunServiceImpl implements AliyunService { + + protected final Logger LOGGER = LoggerFactory.getLogger(this.getClass()); + public static String REGIONID = "cn-hangzhou"; + public static String ACCESSKEYID = "LTAIdaj4eSO79gzz"; + public static String SECRET = "xw6EsvqydZ9HC6zAvIB3ruQqVHynxf"; + public static String APPKEY = "68czunztrb1o"; + + @Override + public AliyunBo addressStand(String address, String city) { + DefaultProfile profile = DefaultProfile.getProfile( + REGIONID, + ACCESSKEYID, + SECRET); + IAcsClient client = new DefaultAcsClient(profile); + StructureAddressRequest request = new StructureAddressRequest(); + request.setServiceCode("addrp"); + request.setAppKey(APPKEY); + request.setText(address); + if(StringUtils.isNotBlank(city)){ + request.setDefaultCity(city); + } + try{ + StructureAddressResponse response = client.getAcsResponse(request); + JSONObject responseJson = JSONObject.fromObject(response.getData()); + String value = (String) responseJson.get("structure"); + AliyunBo aliyunBo = new AliyunBo(); + for (String key:value.split("\t")){ + String[] arry = key.split("="); + wrapField(arry[0],arry[1], aliyunBo); + } + return aliyunBo; + }catch(Exception e){ + LOGGER.error("", e); + return null; + } + } + + private void wrapField(String field,String value, AliyunBo aliyunBo){ + if("city".equals(field)){ + aliyunBo.setCity(value); + }else if("prov".equals(field)){ + aliyunBo.setProv(value); + }else if("district".equals(field)){ + aliyunBo.setDistrict(value); + }else if("poi".equals(field)){ + aliyunBo.setPoi(value); + }else if("subpoi".equals(field)){ + aliyunBo.setSubpoi(value); + }else if("houseno".equals(field)){ + aliyunBo.setHouseno(value); + }else if("cellno".equals(field)){ + aliyunBo.setCellno(value); + }else if("roomno".equals(field)){ + aliyunBo.setRoomno(value); + }else if("road".equals(field)){ + aliyunBo.setRoad(value); + }else if("roadno".equals(field)){ + aliyunBo.setRoadno(value); + }else if("floorno".equals(field)){ + aliyunBo.setFloorno(value); + }else if("town".equals(field)){ + aliyunBo.setTown(value); + }else if("devzone".equals(field)){ + aliyunBo.setDevzone(value); + } + } + public static void main(String[] args) throws ClientException { + DefaultProfile profile = DefaultProfile.getProfile( + REGIONID, + ACCESSKEYID, + SECRET); + IAcsClient client = new DefaultAcsClient(profile); + StructureAddressRequest request = new StructureAddressRequest(); + request.setServiceCode("addrp"); + request.setAppKey(APPKEY); + request.setText("º¼ÖÝÊÐÎÄÒ»Î÷·969ºÅ"); + request.setDefaultProvince("Õã½­Ê¡"); + request.setDefaultCity("º¼ÖÝÊÐ"); + request.setDefaultDistrict("Óàº¼Çø"); + StructureAddressResponse response = client.getAcsResponse(request); + JSONObject responseJson = JSONObject.fromObject(response.getData()); + String value = (String) responseJson.get("structure"); + JSONObject json = new JSONObject(); + for (String key:value.split("\t")){ + String[] arry = key.split("="); + json.put(arry[0],arry[1]); + } + System.out.println(JsonUtils.convertJsonToJavaObj(json, AliyunBo.class)); + } +} diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/bo/AliyunBo.java b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/AliyunBo.java new file mode 100644 index 000000000..e061851d5 --- /dev/null +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/bo/AliyunBo.java @@ -0,0 +1,148 @@ +package com.fangjia.service.bo; + +public class AliyunBo { + private String city; + private String prov; + private String district; + private String poi; + private String houseno; + private String cellno; + private String roomno; + private String status; + private String road; + private String roadno; + private String subroad; + private String floorno; + private String town; + private String subpoi; + private String community; + private String devzone; + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getProv() { + return prov; + } + + public void setProv(String prov) { + this.prov = prov; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getPoi() { + return poi; + } + + public void setPoi(String poi) { + this.poi = poi; + } + + public String getHouseno() { + return houseno; + } + + public void setHouseno(String houseno) { + this.houseno = houseno; + } + + public String getCellno() { + return cellno; + } + + public void setCellno(String cellno) { + this.cellno = cellno; + } + + public String getRoomno() { + return roomno; + } + + public void setRoomno(String roomno) { + this.roomno = roomno; + } + + public String getStatus() { + return status; + } + + public void setStatus(String status) { + this.status = status; + } + + public String getRoad() { + return road; + } + + public void setRoad(String road) { + this.road = road; + } + + public String getRoadno() { + return roadno; + } + + public void setRoadno(String roadno) { + this.roadno = roadno; + } + + public String getSubroad() { + return subroad; + } + + public void setSubroad(String subroad) { + this.subroad = subroad; + } + + public String getFloorno() { + return floorno; + } + + public void setFloorno(String floorno) { + this.floorno = floorno; + } + + public String getTown() { + return town; + } + + public void setTown(String town) { + this.town = town; + } + + public String getSubpoi() { + return subpoi; + } + + public void setSubpoi(String subpoi) { + this.subpoi = subpoi; + } + + public String getCommunity() { + return community; + } + + public void setCommunity(String community) { + this.community = community; + } + + public String getDevzone() { + return devzone; + } + + public void setDevzone(String devzone) { + this.devzone = devzone; + } +} \ No newline at end of file diff --git a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java index 0ee4abde1..a751826c7 100644 --- a/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java +++ b/projects/fangjia-service/src/main/java/com/fangjia/service/mongo/addrepo/AddressRepoServiceImpl.java @@ -128,7 +128,7 @@ public class AddressRepoServiceImpl implements AddressRepoService { } matchMetaDatas.add(data); } - for(String keyword : keywords){ + /*for(String keyword : keywords){ HttpRequest request = new HttpRequest(); Map params = new HashMap(); params.put("query", keyword); @@ -221,7 +221,7 @@ public class AddressRepoServiceImpl implements AddressRepoService { } catch (HttpRequestException e) { e.printStackTrace(); } - } + }*/ Iterator it = matchMetaDatas.iterator(); while(it.hasNext()){ AddressMatchMetaData data = it.next();