From 1f14c987dda241882630edf1b49010af81b8ce53 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sat, 2 Apr 2022 15:03:07 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80=E5=8C=B9?= =?UTF-8?q?=E9=85=8D=E6=89=B9=E9=87=8F=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/BatchTest.java | 1 + src/DivisionBatchTest.java | 1 + src/MatchTest.java | 161 +++++++++++++++++++++ src/RiskLevelTest.java | 1 + src/StructureAndCompleteTest.java | 3 +- src/{ => dto}/FangJiaApiResponse.java | 3 +- src/{ => dto}/FangJiaResponseMsg.java | 1 + src/{ => dto}/FangJiaResponseStatus.java | 2 + src/dto/MatchedBuilding.java | 111 ++++++++++++++ src/{ => dto}/StructureAndCompleteDto.java | 2 + 10 files changed, 284 insertions(+), 2 deletions(-) create mode 100644 src/MatchTest.java rename src/{ => dto}/FangJiaApiResponse.java (96%) rename src/{ => dto}/FangJiaResponseMsg.java (96%) rename src/{ => dto}/FangJiaResponseStatus.java (94%) create mode 100644 src/dto/MatchedBuilding.java rename src/{ => dto}/StructureAndCompleteDto.java (99%) diff --git a/src/BatchTest.java b/src/BatchTest.java index a937249..84bbfcb 100644 --- a/src/BatchTest.java +++ b/src/BatchTest.java @@ -3,6 +3,7 @@ import com.alibaba.excel.ExcelWriter; import com.alibaba.excel.util.StringUtils; import com.alibaba.excel.write.metadata.WriteSheet; import com.alibaba.fastjson.JSONObject; +import dto.FangJiaApiResponse; import org.apache.commons.collections4.CollectionUtils; import org.springframework.util.StopWatch; import org.springframework.web.client.RestTemplate; diff --git a/src/DivisionBatchTest.java b/src/DivisionBatchTest.java index e1b2692..c54d591 100644 --- a/src/DivisionBatchTest.java +++ b/src/DivisionBatchTest.java @@ -2,6 +2,7 @@ import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelWriter; import com.alibaba.excel.write.metadata.WriteSheet; import com.alibaba.fastjson.JSONObject; +import dto.FangJiaApiResponse; import org.apache.commons.collections4.CollectionUtils; import org.springframework.util.StopWatch; import org.springframework.web.client.RestTemplate; diff --git a/src/MatchTest.java b/src/MatchTest.java new file mode 100644 index 0000000..09eeb98 --- /dev/null +++ b/src/MatchTest.java @@ -0,0 +1,161 @@ +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.fastjson.JSONObject; +import dto.FangJiaApiResponse; +import dto.MatchedBuilding; +import dto.StructureAndCompleteDto; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.util.StopWatch; +import org.springframework.web.client.RestTemplate; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 地址匹配批量测试 + */ +public class MatchTest { + + final int column = 2; + + public static void main(String[] args) throws FileNotFoundException, InterruptedException { + MatchTest test = new MatchTest(); + test.run(args[0], args[1], Integer.valueOf(args[2])); + } + + public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException { + ExecutorService executorService = Executors.newFixedThreadPool(nThreads); + Map> dataMap = new ConcurrentHashMap<>(); + AtomicLong count = new AtomicLong(0); + String[] headArr = new String[]{"ID", "地址", "ID", "城市", "区域", "楼盘名称"}; + List> head = this.formatHead(headArr); + // 默认两行数据 + String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx" ; + List> dataList = ExcelUtils.read(new FileInputStream(file)); + if (dataList.size() > 1000000) { + throw new RuntimeException("该批次数据量超过1000000条"); + } + Set set = new HashSet<>(); + dataList.forEach(data -> { + if (data.size() == column ) { + set.add(data.get(column - 1)); + } else { + // 存在为空情况,需保持一致 + set.add(""); + data.add(""); + } + }); + + StopWatch stopWatch = new StopWatch(); + stopWatch.start("总标准化耗时"); + try { + CountDownLatch latch = new CountDownLatch(set.size()); + RestTemplate restTemplate = new RestTemplate(); + for (String addressTemp : set) { + final String address = addressTemp; + executorService.submit(() -> { + try { + String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/match?address={address}&city=重庆市", String.class, address); + FangJiaApiResponse apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class); + if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) { + List addressList = new ArrayList<>(); + MatchedBuilding matchedBuilding = JSONObject.toJavaObject(apiResponse.getResult(), MatchedBuilding.class); + addressList.add(matchedBuilding.getId()); + addressList.add(matchedBuilding.getCity()); + addressList.add(matchedBuilding.getDistrict()); + addressList.add(matchedBuilding.getName()); + dataMap.put(address, addressList); + } + } catch (Exception ex) { + ex.printStackTrace(); + System.out.println("五秒后重试"); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + + } + try { + String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/match?address={address}&city=重庆市", String.class, address); + FangJiaApiResponse apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class); + if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) { + List addressList = new ArrayList<>(); + MatchedBuilding matchedBuilding = JSONObject.toJavaObject(apiResponse.getResult(), MatchedBuilding.class); + addressList.add(matchedBuilding.getId()); + addressList.add(matchedBuilding.getCity()); + addressList.add(matchedBuilding.getDistrict()); + addressList.add(matchedBuilding.getName()); + dataMap.put(address, addressList); + } + } catch (Exception e) { + System.out.println("地址" + address + "异常"); + } + } finally { + latch.countDown(); + count.incrementAndGet(); + synchronized (this) { + if (count.longValue() % 1000 == 0) { + stopWatch.stop(); + System.out.println("已标准化处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() / 60); + stopWatch.start(); + } + } + } + }); + } + latch.await(); + if (stopWatch.isRunning()) { + stopWatch.stop(); + } + System.out.println("已完成该批次地址标准化,耗时" + stopWatch.getTotalTimeSeconds() / 60); + + for (List data : dataList) { + String address = ""; + if (data.size() == column) { + address = data.get(column - 1); + } else { + address = ""; + } + List addressRow = dataMap.get(address); + if (CollectionUtils.isNotEmpty(addressRow)) { + data.addAll(addressRow); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + executorService.shutdown(); + } + + + while (true) { + try { + ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build(); + WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build(); + excelWriter.write(dataList, writeSheet); + excelWriter.finish(); + break; + } catch (Exception ex) { + ex.printStackTrace(); + System.out.println("五秒后重试"); + Thread.sleep(5000); + } + } + } + + private List> formatHead(String[] headArr) { + List> head = new ArrayList<>(); + for (String name: headArr) { + List temp = new ArrayList<>(); + temp.add(name); + head.add(temp); + } + return head; + } +} diff --git a/src/RiskLevelTest.java b/src/RiskLevelTest.java index a383654..1f3ba98 100644 --- a/src/RiskLevelTest.java +++ b/src/RiskLevelTest.java @@ -2,6 +2,7 @@ import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelWriter; import com.alibaba.excel.write.metadata.WriteSheet; import com.alibaba.fastjson.JSONObject; +import dto.FangJiaApiResponse; import org.apache.commons.collections4.CollectionUtils; import org.springframework.web.client.RestTemplate; diff --git a/src/StructureAndCompleteTest.java b/src/StructureAndCompleteTest.java index 542d2fc..d934c44 100644 --- a/src/StructureAndCompleteTest.java +++ b/src/StructureAndCompleteTest.java @@ -1,8 +1,9 @@ import com.alibaba.excel.EasyExcel; import com.alibaba.excel.ExcelWriter; -import com.alibaba.excel.util.StringUtils; import com.alibaba.excel.write.metadata.WriteSheet; import com.alibaba.fastjson.JSONObject; +import dto.FangJiaApiResponse; +import dto.StructureAndCompleteDto; import org.apache.commons.collections4.CollectionUtils; import org.springframework.util.StopWatch; import org.springframework.web.client.RestTemplate; diff --git a/src/FangJiaApiResponse.java b/src/dto/FangJiaApiResponse.java similarity index 96% rename from src/FangJiaApiResponse.java rename to src/dto/FangJiaApiResponse.java index 418afea..afc5f69 100644 --- a/src/FangJiaApiResponse.java +++ b/src/dto/FangJiaApiResponse.java @@ -1,3 +1,4 @@ +package dto; import java.io.Serializable; @@ -100,7 +101,7 @@ public class FangJiaApiResponse implements Serializable { } public static FangJiaApiResponse serverError(){ - return new FangJiaApiResponse<>(FangJiaResponseStatus.FAIL.status,FangJiaResponseMsg.SERVER_ERROR_MSG.msgCode, FangJiaResponseMsg.SERVER_ERROR_MSG.msg); + return new FangJiaApiResponse<>(FangJiaResponseStatus.FAIL.status, FangJiaResponseMsg.SERVER_ERROR_MSG.msgCode, FangJiaResponseMsg.SERVER_ERROR_MSG.msg); } diff --git a/src/FangJiaResponseMsg.java b/src/dto/FangJiaResponseMsg.java similarity index 96% rename from src/FangJiaResponseMsg.java rename to src/dto/FangJiaResponseMsg.java index 4684c0f..b9ba610 100644 --- a/src/FangJiaResponseMsg.java +++ b/src/dto/FangJiaResponseMsg.java @@ -1,3 +1,4 @@ +package dto; public enum FangJiaResponseMsg { diff --git a/src/FangJiaResponseStatus.java b/src/dto/FangJiaResponseStatus.java similarity index 94% rename from src/FangJiaResponseStatus.java rename to src/dto/FangJiaResponseStatus.java index 5d11984..9958b28 100644 --- a/src/FangJiaResponseStatus.java +++ b/src/dto/FangJiaResponseStatus.java @@ -1,3 +1,5 @@ +package dto; + public enum FangJiaResponseStatus { OK(0, "ok"), FAIL(1, "失败"); diff --git a/src/dto/MatchedBuilding.java b/src/dto/MatchedBuilding.java new file mode 100644 index 0000000..05cf1ba --- /dev/null +++ b/src/dto/MatchedBuilding.java @@ -0,0 +1,111 @@ +package dto; + +import org.springframework.util.StringUtils; + +import java.util.StringJoiner; + +public class MatchedBuilding { + + private String id; + private String city; + private String district; + private String name; + private String address; + private Double lng; + private Double lat; + private String propertyType; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getAddress() { + return address; + } + + public void setAddress(String address) { + this.address = address; + } + + public Double getLng() { + return lng; + } + + public void setLng(Double lng) { + this.lng = lng; + } + + public Double getLat() { + return lat; + } + + public void setLat(Double lat) { + this.lat = lat; + } + + public String getPropertyType() { + return propertyType; + } + + public void setPropertyType(String propertyType) { + this.propertyType = propertyType; + } + + @Override + public String toString() { + StringJoiner stringJoiner = new StringJoiner(" "); + if (!StringUtils.isEmpty(city)) { + stringJoiner.add("城市:" + city); + } + if (!StringUtils.isEmpty(district)) { + stringJoiner.add("区域:" + district); + } + if (!StringUtils.isEmpty(id)) { + stringJoiner.add("楼盘ID:" + id); + } + if (!StringUtils.isEmpty(name)) { + stringJoiner.add("楼盘名称:" + name); + } + if (!StringUtils.isEmpty(address)) { + stringJoiner.add("地址:" + address); + } + if (!StringUtils.isEmpty(propertyType)) { + stringJoiner.add("物业类型:" + propertyType); + } + if (lng != null) { + stringJoiner.add("经度:" + lng); + } + if (lat != null) { + stringJoiner.add("纬度:" + lat); + } + return stringJoiner.toString(); + } +} diff --git a/src/StructureAndCompleteDto.java b/src/dto/StructureAndCompleteDto.java similarity index 99% rename from src/StructureAndCompleteDto.java rename to src/dto/StructureAndCompleteDto.java index f6b93e4..1b20879 100644 --- a/src/StructureAndCompleteDto.java +++ b/src/dto/StructureAndCompleteDto.java @@ -1,3 +1,5 @@ +package dto; + public class StructureAndCompleteDto { private String province;