新增地址匹配批量测试

This commit is contained in:
Jian Shen
2022-04-02 15:03:07 +08:00
parent ac16af6c7c
commit 1f14c987dd
10 changed files with 284 additions and 2 deletions
+1
View File
@@ -3,6 +3,7 @@ import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import dto.FangJiaApiResponse;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.util.StopWatch;
import org.springframework.web.client.RestTemplate;
+1
View File
@@ -2,6 +2,7 @@ import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import dto.FangJiaApiResponse;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.util.StopWatch;
import org.springframework.web.client.RestTemplate;
+161
View File
@@ -0,0 +1,161 @@
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import dto.FangJiaApiResponse;
import dto.MatchedBuilding;
import dto.StructureAndCompleteDto;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.util.StopWatch;
import org.springframework.web.client.RestTemplate;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
/**
* 地址匹配批量测试
*/
public class MatchTest {
final int column = 2;
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
MatchTest test = new MatchTest();
test.run(args[0], args[1], Integer.valueOf(args[2]));
}
public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException {
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
AtomicLong count = new AtomicLong(0);
String[] headArr = new String[]{"ID", "地址", "ID", "城市", "区域", "楼盘名称"};
List<List<String>> head = this.formatHead(headArr);
// 默认两行数据
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx" ;
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
if (dataList.size() > 1000000) {
throw new RuntimeException("该批次数据量超过1000000条");
}
Set<String> set = new HashSet<>();
dataList.forEach(data -> {
if (data.size() == column ) {
set.add(data.get(column - 1));
} else {
// 存在为空情况,需保持一致
set.add("");
data.add("");
}
});
StopWatch stopWatch = new StopWatch();
stopWatch.start("总标准化耗时");
try {
CountDownLatch latch = new CountDownLatch(set.size());
RestTemplate restTemplate = new RestTemplate();
for (String addressTemp : set) {
final String address = addressTemp;
executorService.submit(() -> {
try {
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/match?address={address}&city=重庆市", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> addressList = new ArrayList<>();
MatchedBuilding matchedBuilding = JSONObject.toJavaObject(apiResponse.getResult(), MatchedBuilding.class);
addressList.add(matchedBuilding.getId());
addressList.add(matchedBuilding.getCity());
addressList.add(matchedBuilding.getDistrict());
addressList.add(matchedBuilding.getName());
dataMap.put(address, addressList);
}
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
try {
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/match?address={address}&city=重庆市", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> addressList = new ArrayList<>();
MatchedBuilding matchedBuilding = JSONObject.toJavaObject(apiResponse.getResult(), MatchedBuilding.class);
addressList.add(matchedBuilding.getId());
addressList.add(matchedBuilding.getCity());
addressList.add(matchedBuilding.getDistrict());
addressList.add(matchedBuilding.getName());
dataMap.put(address, addressList);
}
} catch (Exception e) {
System.out.println("地址" + address + "异常");
}
} finally {
latch.countDown();
count.incrementAndGet();
synchronized (this) {
if (count.longValue() % 1000 == 0) {
stopWatch.stop();
System.out.println("已标准化处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() / 60);
stopWatch.start();
}
}
}
});
}
latch.await();
if (stopWatch.isRunning()) {
stopWatch.stop();
}
System.out.println("已完成该批次地址标准化,耗时" + stopWatch.getTotalTimeSeconds() / 60);
for (List<String> data : dataList) {
String address = "";
if (data.size() == column) {
address = data.get(column - 1);
} else {
address = "";
}
List<String> addressRow = dataMap.get(address);
if (CollectionUtils.isNotEmpty(addressRow)) {
data.addAll(addressRow);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
executorService.shutdown();
}
while (true) {
try {
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build();
excelWriter.write(dataList, writeSheet);
excelWriter.finish();
break;
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
Thread.sleep(5000);
}
}
}
private List<List<String>> formatHead(String[] headArr) {
List<List<String>> head = new ArrayList<>();
for (String name: headArr) {
List<String> temp = new ArrayList<>();
temp.add(name);
head.add(temp);
}
return head;
}
}
+1
View File
@@ -2,6 +2,7 @@ import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import dto.FangJiaApiResponse;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.web.client.RestTemplate;
+2 -1
View File
@@ -1,8 +1,9 @@
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import dto.FangJiaApiResponse;
import dto.StructureAndCompleteDto;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.util.StopWatch;
import org.springframework.web.client.RestTemplate;
@@ -1,3 +1,4 @@
package dto;
import java.io.Serializable;
@@ -100,7 +101,7 @@ public class FangJiaApiResponse<T> implements Serializable {
}
public static FangJiaApiResponse<Object> serverError(){
return new FangJiaApiResponse<>(FangJiaResponseStatus.FAIL.status,FangJiaResponseMsg.SERVER_ERROR_MSG.msgCode, FangJiaResponseMsg.SERVER_ERROR_MSG.msg);
return new FangJiaApiResponse<>(FangJiaResponseStatus.FAIL.status, FangJiaResponseMsg.SERVER_ERROR_MSG.msgCode, FangJiaResponseMsg.SERVER_ERROR_MSG.msg);
}
@@ -1,3 +1,4 @@
package dto;
public enum FangJiaResponseMsg {
@@ -1,3 +1,5 @@
package dto;
public enum FangJiaResponseStatus {
OK(0, "ok"), FAIL(1, "失败");
+111
View File
@@ -0,0 +1,111 @@
package dto;
import org.springframework.util.StringUtils;
import java.util.StringJoiner;
public class MatchedBuilding {
private String id;
private String city;
private String district;
private String name;
private String address;
private Double lng;
private Double lat;
private String propertyType;
public String getId() {
return id;
}
public void setId(String id) {
this.id = id;
}
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getAddress() {
return address;
}
public void setAddress(String address) {
this.address = address;
}
public Double getLng() {
return lng;
}
public void setLng(Double lng) {
this.lng = lng;
}
public Double getLat() {
return lat;
}
public void setLat(Double lat) {
this.lat = lat;
}
public String getPropertyType() {
return propertyType;
}
public void setPropertyType(String propertyType) {
this.propertyType = propertyType;
}
@Override
public String toString() {
StringJoiner stringJoiner = new StringJoiner(" ");
if (!StringUtils.isEmpty(city)) {
stringJoiner.add("城市:" + city);
}
if (!StringUtils.isEmpty(district)) {
stringJoiner.add("区域:" + district);
}
if (!StringUtils.isEmpty(id)) {
stringJoiner.add("楼盘ID:" + id);
}
if (!StringUtils.isEmpty(name)) {
stringJoiner.add("楼盘名称:" + name);
}
if (!StringUtils.isEmpty(address)) {
stringJoiner.add("地址:" + address);
}
if (!StringUtils.isEmpty(propertyType)) {
stringJoiner.add("物业类型:" + propertyType);
}
if (lng != null) {
stringJoiner.add("经度:" + lng);
}
if (lat != null) {
stringJoiner.add("纬度:" + lat);
}
return stringJoiner.toString();
}
}
@@ -1,3 +1,5 @@
package dto;
public class StructureAndCompleteDto {
private String province;