Files
address-batch-utils/src/VillageIdentifyTest.java
2024-05-11 17:44:38 +08:00

209 lines
11 KiB
Java

import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import dto.AddressResultDto;
import dto.FangJiaApiResponse;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.util.StopWatch;
import org.springframework.web.client.RestTemplate;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.text.SimpleDateFormat;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
public class VillageIdentifyTest {
static final String IP = "192.168.0.100:32080";
static final String sourceFilePath = "E:\\download\\跑批测试.xlsx";
static final int column = 2;
//涉农类型,1:对私,2:对公
static final Integer nThreads = 10;
static final String type = "2";
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
VillageIdentifyTest test = new VillageIdentifyTest();
test.run(IP, sourceFilePath, nThreads);
}
public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException {
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
String formattedDate = sdf.format(new Date());
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
AtomicLong count = new AtomicLong(0);
String[] headArr = new String[]{
"编号", "地址", "省份", "省份编码", "城市", "城市编码", "区域", "区域编码", "乡镇街道", "乡镇街道编码", "居委村委", "居委村委编码", "城乡分类编码",
"城乡分类", "涉农标识", "地址验证", "是否县级市", "是否市辖建制镇", "是否开发区", "开发区名称", "开发区等级", "是否县政府"};
List<List<String>> head = this.formatHead(headArr);
// 默认两行数据
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx";
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
if (dataList.size() > 1000000) {
throw new RuntimeException("该批次数据量超过1000000条");
}
Set<String> set = new HashSet<>();
dataList.forEach(data -> {
if (data.size() == column) {
set.add(data.get(column - 1));
} else {
// 存在为空情况,需保持一致
set.add("");
data.add("");
}
});
StopWatch stopWatch = new StopWatch();
stopWatch.start("总耗时");
System.out.println("开始时间:"+ formattedDate);
try {
CountDownLatch latch = new CountDownLatch(set.size());
RestTemplate restTemplate = new RestTemplate();
for (String addressTemp : set) {
final String address = addressTemp;
executorService.submit(() -> {
try {
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/villageIdentify?address={address}&type={type}&showFullValue=2", String.class, address, type);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> result = new ArrayList<>();
AddressResultDto villageIdentifyResultDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class);
result.add(villageIdentifyResultDto.getProvince());
result.add(villageIdentifyResultDto.getProvinceCode());
result.add(villageIdentifyResultDto.getCity());
result.add(villageIdentifyResultDto.getCityCode());
result.add(villageIdentifyResultDto.getDistrict());
result.add(villageIdentifyResultDto.getDistrictCode());
result.add(villageIdentifyResultDto.getTown());
result.add(villageIdentifyResultDto.getTownCode());
result.add(villageIdentifyResultDto.getCommittee());
result.add(villageIdentifyResultDto.getCommitteeCode());
result.add(villageIdentifyResultDto.getTypeCode());
result.add(villageIdentifyResultDto.getTypeName());
result.add(villageIdentifyResultDto.getVerifyFlag());
result.add(villageIdentifyResultDto.getAddrVerify());
result.add(villageIdentifyResultDto.getCountyLevelCity());
result.add(villageIdentifyResultDto.getAdministrativeTowns());
result.add(villageIdentifyResultDto.getDevDistrict());
result.add(villageIdentifyResultDto.getDevDistrictName());
result.add(villageIdentifyResultDto.getDevLevel());
result.add(villageIdentifyResultDto.getCountyGov());
dataMap.put(address, result);
}
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
try {
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/villageIdentify?address={address}&type={type}", String.class, address, type);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> result = new ArrayList<>();
AddressResultDto villageIdentifyResultDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class);
result.add(villageIdentifyResultDto.getProvince());
result.add(villageIdentifyResultDto.getProvinceCode());
result.add(villageIdentifyResultDto.getCity());
result.add(villageIdentifyResultDto.getCityCode());
result.add(villageIdentifyResultDto.getDistrict());
result.add(villageIdentifyResultDto.getDistrictCode());
result.add(villageIdentifyResultDto.getTown());
result.add(villageIdentifyResultDto.getTownCode());
result.add(villageIdentifyResultDto.getCommittee());
result.add(villageIdentifyResultDto.getCommitteeCode());
result.add(villageIdentifyResultDto.getTypeCode());
result.add(villageIdentifyResultDto.getTypeName());
result.add(villageIdentifyResultDto.getVerifyFlag());
result.add(villageIdentifyResultDto.getAddrVerify());
result.add(villageIdentifyResultDto.getCountyLevelCity());
result.add(villageIdentifyResultDto.getAdministrativeTowns());
result.add(villageIdentifyResultDto.getDevDistrict());
result.add(villageIdentifyResultDto.getDevDistrictName());
result.add(villageIdentifyResultDto.getDevLevel());
result.add(villageIdentifyResultDto.getCountyGov());
dataMap.put(address, result);
}
} catch (Exception e) {
System.out.println("地址" + address + "异常");
}
} finally {
latch.countDown();
count.incrementAndGet();
synchronized (this) {
if (count.longValue() % 1000 == 0) {
stopWatch.stop();
System.out.println("已处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds()+"秒");
stopWatch.start();
}
}
}
});
}
latch.await();
if (stopWatch.isRunning()) {
stopWatch.stop();
}
System.out.println("已完成该批次涉农识别,耗时" + stopWatch.getTotalTimeSeconds()/60+"分钟");
System.out.println("完成时间:" + sdf.format(new Date()));
for (List<String> data : dataList) {
String address = "";
if (data.size() == column) {
address = data.get(column - 1);
} else {
address = "";
}
List<String> addressRow = dataMap.get(address);
if (CollectionUtils.isNotEmpty(addressRow)) {
data.addAll(addressRow);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
executorService.shutdown();
}
while (true) {
try {
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "涉农结果").head(head).build();
excelWriter.write(dataList, writeSheet);
excelWriter.finish();
break;
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
Thread.sleep(5000);
}
}
}
private List<List<String>> formatHead(String[] headArr) {
List<List<String>> head = new ArrayList<>();
for (String name : headArr) {
List<String> temp = new ArrayList<>();
temp.add(name);
head.add(temp);
}
return head;
}
}