优化批量
This commit is contained in:
+17
-11
@@ -21,27 +21,25 @@ import java.util.concurrent.atomic.AtomicLong;
|
|||||||
|
|
||||||
public class BatchTest {
|
public class BatchTest {
|
||||||
|
|
||||||
static final String IP = "192.168.0.216:39080";
|
|
||||||
final String sourceFilePath = "D:\\goojia\\批量测试10万.xlsx";
|
|
||||||
final String aimFilePath = "D:\\goojia\\批量测试10万结果.xlsx";
|
|
||||||
final int column = 2;
|
final int column = 2;
|
||||||
final Integer nThreads = 20;
|
final Integer nThreads = 20;
|
||||||
|
|
||||||
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
||||||
BatchTest test = new BatchTest();
|
BatchTest test = new BatchTest();
|
||||||
test.run();
|
test.run(args[0], args[1]);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void run() throws FileNotFoundException, InterruptedException {
|
public void run(String ip, String file) throws FileNotFoundException, InterruptedException {
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
||||||
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
||||||
AtomicLong count = new AtomicLong(0);
|
AtomicLong count = new AtomicLong(0);
|
||||||
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证"};
|
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证", "经度", "维度", "百度经度", "百度纬度"};
|
||||||
List<List<String>> head = this.formatHead(headArr);
|
List<List<String>> head = this.formatHead(headArr);
|
||||||
// 默认两行数据
|
// 默认两行数据
|
||||||
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(sourceFilePath));
|
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx" ;
|
||||||
if (dataList.size() > 100000) {
|
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
|
||||||
throw new RuntimeException("该批次数据量超过100000条");
|
if (dataList.size() > 1000000) {
|
||||||
|
throw new RuntimeException("该批次数据量超过1000000条");
|
||||||
}
|
}
|
||||||
Set<String> set = new HashSet<>();
|
Set<String> set = new HashSet<>();
|
||||||
dataList.forEach(data -> {
|
dataList.forEach(data -> {
|
||||||
@@ -63,7 +61,7 @@ public class BatchTest {
|
|||||||
final String address = addressTemp;
|
final String address = addressTemp;
|
||||||
executorService.submit(() -> {
|
executorService.submit(() -> {
|
||||||
try {
|
try {
|
||||||
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
|
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}&city=江阴市", String.class, address);
|
||||||
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
List<String> addressList = new ArrayList<>();
|
List<String> addressList = new ArrayList<>();
|
||||||
@@ -77,6 +75,10 @@ public class BatchTest {
|
|||||||
addressList.add(structuredAddress.getFormatAddress());
|
addressList.add(structuredAddress.getFormatAddress());
|
||||||
addressList.add(structuredAddress.getIsAgriculture());
|
addressList.add(structuredAddress.getIsAgriculture());
|
||||||
addressList.add(structuredAddress.getAddrVerify());
|
addressList.add(structuredAddress.getAddrVerify());
|
||||||
|
addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : "");
|
||||||
|
addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : "");
|
||||||
|
addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : "");
|
||||||
|
addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : "");
|
||||||
dataMap.put(address, addressList);
|
dataMap.put(address, addressList);
|
||||||
}
|
}
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
@@ -87,7 +89,7 @@ public class BatchTest {
|
|||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
|
|
||||||
}
|
}
|
||||||
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
|
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}&city=江阴市", String.class, address);
|
||||||
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
try {
|
try {
|
||||||
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
@@ -102,6 +104,10 @@ public class BatchTest {
|
|||||||
addressList.add(structuredAddress.getFormatAddress());
|
addressList.add(structuredAddress.getFormatAddress());
|
||||||
addressList.add(structuredAddress.getIsAgriculture());
|
addressList.add(structuredAddress.getIsAgriculture());
|
||||||
addressList.add(structuredAddress.getAddrVerify());
|
addressList.add(structuredAddress.getAddrVerify());
|
||||||
|
addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : "");
|
||||||
|
addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : "");
|
||||||
|
addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : "");
|
||||||
|
addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : "");
|
||||||
dataMap.put(address, addressList);
|
dataMap.put(address, addressList);
|
||||||
}
|
}
|
||||||
} catch (Exception e) {
|
} catch (Exception e) {
|
||||||
|
|||||||
Reference in New Issue
Block a user