新增地址补全批量测试
This commit is contained in:
@@ -0,0 +1,138 @@
|
|||||||
|
public class StructureAndCompleteDto {
|
||||||
|
|
||||||
|
private String province;
|
||||||
|
private String city;
|
||||||
|
private String district;
|
||||||
|
private String dev;
|
||||||
|
private String township;
|
||||||
|
private String community;
|
||||||
|
private String street;
|
||||||
|
private String subStreet;
|
||||||
|
private String streetNumber;
|
||||||
|
private String name;
|
||||||
|
private String group;
|
||||||
|
private String buildingNumber;
|
||||||
|
private String unitNumber;
|
||||||
|
private String floor;
|
||||||
|
private String roomNumber;
|
||||||
|
|
||||||
|
public String getProvince() {
|
||||||
|
return province;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProvince(String province) {
|
||||||
|
this.province = province;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCity() {
|
||||||
|
return city;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCity(String city) {
|
||||||
|
this.city = city;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDistrict() {
|
||||||
|
return district;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDistrict(String district) {
|
||||||
|
this.district = district;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDev() {
|
||||||
|
return dev;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDev(String dev) {
|
||||||
|
this.dev = dev;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTownship() {
|
||||||
|
return township;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTownship(String township) {
|
||||||
|
this.township = township;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCommunity() {
|
||||||
|
return community;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommunity(String community) {
|
||||||
|
this.community = community;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getStreet() {
|
||||||
|
return street;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setStreet(String street) {
|
||||||
|
this.street = street;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSubStreet() {
|
||||||
|
return subStreet;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSubStreet(String subStreet) {
|
||||||
|
this.subStreet = subStreet;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getStreetNumber() {
|
||||||
|
return streetNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setStreetNumber(String streetNumber) {
|
||||||
|
this.streetNumber = streetNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setName(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getGroup() {
|
||||||
|
return group;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setGroup(String group) {
|
||||||
|
this.group = group;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getBuildingNumber() {
|
||||||
|
return buildingNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBuildingNumber(String buildingNumber) {
|
||||||
|
this.buildingNumber = buildingNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getUnitNumber() {
|
||||||
|
return unitNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setUnitNumber(String unitNumber) {
|
||||||
|
this.unitNumber = unitNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getFloor() {
|
||||||
|
return floor;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFloor(String floor) {
|
||||||
|
this.floor = floor;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getRoomNumber() {
|
||||||
|
return roomNumber;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setRoomNumber(String roomNumber) {
|
||||||
|
this.roomNumber = roomNumber;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,179 @@
|
|||||||
|
import com.alibaba.excel.EasyExcel;
|
||||||
|
import com.alibaba.excel.ExcelWriter;
|
||||||
|
import com.alibaba.excel.util.StringUtils;
|
||||||
|
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
|
import org.springframework.util.StopWatch;
|
||||||
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 地址补全批量测试
|
||||||
|
*/
|
||||||
|
public class StructureAndCompleteTest {
|
||||||
|
|
||||||
|
final int column = 2;
|
||||||
|
|
||||||
|
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
||||||
|
StructureAndCompleteTest test = new StructureAndCompleteTest();
|
||||||
|
test.run(args[0], args[1], Integer.valueOf(args[2]));
|
||||||
|
}
|
||||||
|
|
||||||
|
public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException {
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
||||||
|
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
||||||
|
AtomicLong count = new AtomicLong(0);
|
||||||
|
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "居委村委", "道路", "子路", "路号", "名称", "村组", "楼栋号", "单元号", "楼层", "室号"};
|
||||||
|
List<List<String>> head = this.formatHead(headArr);
|
||||||
|
// 默认两行数据
|
||||||
|
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx" ;
|
||||||
|
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
|
||||||
|
if (dataList.size() > 1000000) {
|
||||||
|
throw new RuntimeException("该批次数据量超过1000000条");
|
||||||
|
}
|
||||||
|
Set<String> set = new HashSet<>();
|
||||||
|
dataList.forEach(data -> {
|
||||||
|
if (data.size() == column ) {
|
||||||
|
set.add(data.get(column - 1));
|
||||||
|
} else {
|
||||||
|
// 存在为空情况,需保持一致
|
||||||
|
set.add("");
|
||||||
|
data.add("");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
StopWatch stopWatch = new StopWatch();
|
||||||
|
stopWatch.start("总标准化耗时");
|
||||||
|
try {
|
||||||
|
CountDownLatch latch = new CountDownLatch(set.size());
|
||||||
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
|
for (String addressTemp : set) {
|
||||||
|
final String address = addressTemp;
|
||||||
|
executorService.submit(() -> {
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/structureAndComplete?address={address}&city=重庆市", String.class, address);
|
||||||
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> addressList = new ArrayList<>();
|
||||||
|
StructureAndCompleteDto structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), StructureAndCompleteDto.class);
|
||||||
|
addressList.add(structuredAddress.getProvince());
|
||||||
|
addressList.add(structuredAddress.getCity());
|
||||||
|
addressList.add(structuredAddress.getDistrict());
|
||||||
|
addressList.add(structuredAddress.getTownship());
|
||||||
|
addressList.add(structuredAddress.getCommunity());
|
||||||
|
addressList.add(structuredAddress.getStreet());
|
||||||
|
addressList.add(structuredAddress.getSubStreet());
|
||||||
|
addressList.add(structuredAddress.getStreetNumber());
|
||||||
|
addressList.add(structuredAddress.getName());
|
||||||
|
addressList.add(structuredAddress.getGroup());
|
||||||
|
addressList.add(structuredAddress.getBuildingNumber());
|
||||||
|
addressList.add(structuredAddress.getUnitNumber());
|
||||||
|
addressList.add(structuredAddress.getFloor());
|
||||||
|
addressList.add(structuredAddress.getRoomNumber());
|
||||||
|
dataMap.put(address, addressList);
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
try {
|
||||||
|
Thread.sleep(5000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/structureAndComplete?address={address}&city=重庆市", String.class, address);
|
||||||
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> addressList = new ArrayList<>();
|
||||||
|
StructureAndCompleteDto structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), StructureAndCompleteDto.class);
|
||||||
|
addressList.add(structuredAddress.getProvince());
|
||||||
|
addressList.add(structuredAddress.getCity());
|
||||||
|
addressList.add(structuredAddress.getDistrict());
|
||||||
|
addressList.add(structuredAddress.getTownship());
|
||||||
|
addressList.add(structuredAddress.getCommunity());
|
||||||
|
addressList.add(structuredAddress.getStreet());
|
||||||
|
addressList.add(structuredAddress.getSubStreet());
|
||||||
|
addressList.add(structuredAddress.getStreetNumber());
|
||||||
|
addressList.add(structuredAddress.getName());
|
||||||
|
addressList.add(structuredAddress.getGroup());
|
||||||
|
addressList.add(structuredAddress.getBuildingNumber());
|
||||||
|
addressList.add(structuredAddress.getUnitNumber());
|
||||||
|
addressList.add(structuredAddress.getFloor());
|
||||||
|
addressList.add(structuredAddress.getRoomNumber());
|
||||||
|
dataMap.put(address, addressList);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println("地址" + address + "异常");
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
latch.countDown();
|
||||||
|
count.incrementAndGet();
|
||||||
|
synchronized (this) {
|
||||||
|
if (count.longValue() % 1000 == 0) {
|
||||||
|
stopWatch.stop();
|
||||||
|
System.out.println("已标准化处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() / 60);
|
||||||
|
stopWatch.start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await();
|
||||||
|
if (stopWatch.isRunning()) {
|
||||||
|
stopWatch.stop();
|
||||||
|
}
|
||||||
|
System.out.println("已完成该批次地址标准化,耗时" + stopWatch.getTotalTimeSeconds() / 60);
|
||||||
|
|
||||||
|
for (List<String> data : dataList) {
|
||||||
|
String address = "";
|
||||||
|
if (data.size() == column) {
|
||||||
|
address = data.get(column - 1);
|
||||||
|
} else {
|
||||||
|
address = "";
|
||||||
|
}
|
||||||
|
List<String> addressRow = dataMap.get(address);
|
||||||
|
if (CollectionUtils.isNotEmpty(addressRow)) {
|
||||||
|
data.addAll(addressRow);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} finally {
|
||||||
|
executorService.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
|
||||||
|
WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build();
|
||||||
|
excelWriter.write(dataList, writeSheet);
|
||||||
|
excelWriter.finish();
|
||||||
|
break;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
Thread.sleep(5000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private List<List<String>> formatHead(String[] headArr) {
|
||||||
|
List<List<String>> head = new ArrayList<>();
|
||||||
|
for (String name: headArr) {
|
||||||
|
List<String> temp = new ArrayList<>();
|
||||||
|
temp.add(name);
|
||||||
|
head.add(temp);
|
||||||
|
}
|
||||||
|
return head;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user