优化批量

This commit is contained in:
Jian Shen
2022-02-17 17:02:37 +08:00
parent 27648a1b55
commit e8905e75b8
+17 -11
View File
@@ -21,27 +21,25 @@ import java.util.concurrent.atomic.AtomicLong;
public class BatchTest {
static final String IP = "192.168.0.216:39080";
final String sourceFilePath = "D:\\goojia\\批量测试10万.xlsx";
final String aimFilePath = "D:\\goojia\\批量测试10万结果.xlsx";
final int column = 2;
final Integer nThreads = 20;
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
BatchTest test = new BatchTest();
test.run();
test.run(args[0], args[1]);
}
public void run() throws FileNotFoundException, InterruptedException {
public void run(String ip, String file) throws FileNotFoundException, InterruptedException {
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
AtomicLong count = new AtomicLong(0);
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证"};
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证", "经度", "维度", "百度经度", "百度纬度"};
List<List<String>> head = this.formatHead(headArr);
// 默认两行数据
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(sourceFilePath));
if (dataList.size() > 100000) {
throw new RuntimeException("该批次数据量超过100000条");
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx" ;
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
if (dataList.size() > 1000000) {
throw new RuntimeException("该批次数据量超过1000000条");
}
Set<String> set = new HashSet<>();
dataList.forEach(data -> {
@@ -63,7 +61,7 @@ public class BatchTest {
final String address = addressTemp;
executorService.submit(() -> {
try {
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}&city=江阴市", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> addressList = new ArrayList<>();
@@ -77,6 +75,10 @@ public class BatchTest {
addressList.add(structuredAddress.getFormatAddress());
addressList.add(structuredAddress.getIsAgriculture());
addressList.add(structuredAddress.getAddrVerify());
addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : "");
addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : "");
addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : "");
addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : "");
dataMap.put(address, addressList);
}
} catch (Exception ex) {
@@ -87,7 +89,7 @@ public class BatchTest {
} catch (InterruptedException e) {
}
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}&city=江阴市", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
try {
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
@@ -102,6 +104,10 @@ public class BatchTest {
addressList.add(structuredAddress.getFormatAddress());
addressList.add(structuredAddress.getIsAgriculture());
addressList.add(structuredAddress.getAddrVerify());
addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : "");
addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : "");
addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : "");
addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : "");
dataMap.put(address, addressList);
}
} catch (Exception e) {