diff --git a/src/StructureAndCompleteDto.java b/src/StructureAndCompleteDto.java new file mode 100644 index 0000000..f6b93e4 --- /dev/null +++ b/src/StructureAndCompleteDto.java @@ -0,0 +1,138 @@ +public class StructureAndCompleteDto { + + private String province; + private String city; + private String district; + private String dev; + private String township; + private String community; + private String street; + private String subStreet; + private String streetNumber; + private String name; + private String group; + private String buildingNumber; + private String unitNumber; + private String floor; + private String roomNumber; + + public String getProvince() { + return province; + } + + public void setProvince(String province) { + this.province = province; + } + + public String getCity() { + return city; + } + + public void setCity(String city) { + this.city = city; + } + + public String getDistrict() { + return district; + } + + public void setDistrict(String district) { + this.district = district; + } + + public String getDev() { + return dev; + } + + public void setDev(String dev) { + this.dev = dev; + } + + public String getTownship() { + return township; + } + + public void setTownship(String township) { + this.township = township; + } + + public String getCommunity() { + return community; + } + + public void setCommunity(String community) { + this.community = community; + } + + public String getStreet() { + return street; + } + + public void setStreet(String street) { + this.street = street; + } + + public String getSubStreet() { + return subStreet; + } + + public void setSubStreet(String subStreet) { + this.subStreet = subStreet; + } + + public String getStreetNumber() { + return streetNumber; + } + + public void setStreetNumber(String streetNumber) { + this.streetNumber = streetNumber; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getGroup() { + return group; + } + + public void setGroup(String group) { + this.group = group; + } + + public String getBuildingNumber() { + return buildingNumber; + } + + public void setBuildingNumber(String buildingNumber) { + this.buildingNumber = buildingNumber; + } + + public String getUnitNumber() { + return unitNumber; + } + + public void setUnitNumber(String unitNumber) { + this.unitNumber = unitNumber; + } + + public String getFloor() { + return floor; + } + + public void setFloor(String floor) { + this.floor = floor; + } + + public String getRoomNumber() { + return roomNumber; + } + + public void setRoomNumber(String roomNumber) { + this.roomNumber = roomNumber; + } +} diff --git a/src/StructureAndCompleteTest.java b/src/StructureAndCompleteTest.java new file mode 100644 index 0000000..542d2fc --- /dev/null +++ b/src/StructureAndCompleteTest.java @@ -0,0 +1,179 @@ +import com.alibaba.excel.EasyExcel; +import com.alibaba.excel.ExcelWriter; +import com.alibaba.excel.util.StringUtils; +import com.alibaba.excel.write.metadata.WriteSheet; +import com.alibaba.fastjson.JSONObject; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.util.StopWatch; +import org.springframework.web.client.RestTemplate; + +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.util.*; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicLong; + +/** + * 地址补全批量测试 + */ +public class StructureAndCompleteTest { + + final int column = 2; + + public static void main(String[] args) throws FileNotFoundException, InterruptedException { + StructureAndCompleteTest test = new StructureAndCompleteTest(); + test.run(args[0], args[1], Integer.valueOf(args[2])); + } + + public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException { + ExecutorService executorService = Executors.newFixedThreadPool(nThreads); + Map> dataMap = new ConcurrentHashMap<>(); + AtomicLong count = new AtomicLong(0); + String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "居委村委", "道路", "子路", "路号", "名称", "村组", "楼栋号", "单元号", "楼层", "室号"}; + List> head = this.formatHead(headArr); + // 默认两行数据 + String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx" ; + List> dataList = ExcelUtils.read(new FileInputStream(file)); + if (dataList.size() > 1000000) { + throw new RuntimeException("该批次数据量超过1000000条"); + } + Set set = new HashSet<>(); + dataList.forEach(data -> { + if (data.size() == column ) { + set.add(data.get(column - 1)); + } else { + // 存在为空情况,需保持一致 + set.add(""); + data.add(""); + } + }); + + StopWatch stopWatch = new StopWatch(); + stopWatch.start("总标准化耗时"); + try { + CountDownLatch latch = new CountDownLatch(set.size()); + RestTemplate restTemplate = new RestTemplate(); + for (String addressTemp : set) { + final String address = addressTemp; + executorService.submit(() -> { + try { + String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/structureAndComplete?address={address}&city=重庆市", String.class, address); + FangJiaApiResponse apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class); + if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) { + List addressList = new ArrayList<>(); + StructureAndCompleteDto structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), StructureAndCompleteDto.class); + addressList.add(structuredAddress.getProvince()); + addressList.add(structuredAddress.getCity()); + addressList.add(structuredAddress.getDistrict()); + addressList.add(structuredAddress.getTownship()); + addressList.add(structuredAddress.getCommunity()); + addressList.add(structuredAddress.getStreet()); + addressList.add(structuredAddress.getSubStreet()); + addressList.add(structuredAddress.getStreetNumber()); + addressList.add(structuredAddress.getName()); + addressList.add(structuredAddress.getGroup()); + addressList.add(structuredAddress.getBuildingNumber()); + addressList.add(structuredAddress.getUnitNumber()); + addressList.add(structuredAddress.getFloor()); + addressList.add(structuredAddress.getRoomNumber()); + dataMap.put(address, addressList); + } + } catch (Exception ex) { + ex.printStackTrace(); + System.out.println("五秒后重试"); + try { + Thread.sleep(5000); + } catch (InterruptedException e) { + + } + try { + String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/structureAndComplete?address={address}&city=重庆市", String.class, address); + FangJiaApiResponse apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class); + if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) { + List addressList = new ArrayList<>(); + StructureAndCompleteDto structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), StructureAndCompleteDto.class); + addressList.add(structuredAddress.getProvince()); + addressList.add(structuredAddress.getCity()); + addressList.add(structuredAddress.getDistrict()); + addressList.add(structuredAddress.getTownship()); + addressList.add(structuredAddress.getCommunity()); + addressList.add(structuredAddress.getStreet()); + addressList.add(structuredAddress.getSubStreet()); + addressList.add(structuredAddress.getStreetNumber()); + addressList.add(structuredAddress.getName()); + addressList.add(structuredAddress.getGroup()); + addressList.add(structuredAddress.getBuildingNumber()); + addressList.add(structuredAddress.getUnitNumber()); + addressList.add(structuredAddress.getFloor()); + addressList.add(structuredAddress.getRoomNumber()); + dataMap.put(address, addressList); + } + } catch (Exception e) { + System.out.println("地址" + address + "异常"); + } + } finally { + latch.countDown(); + count.incrementAndGet(); + synchronized (this) { + if (count.longValue() % 1000 == 0) { + stopWatch.stop(); + System.out.println("已标准化处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() / 60); + stopWatch.start(); + } + } + } + }); + } + latch.await(); + if (stopWatch.isRunning()) { + stopWatch.stop(); + } + System.out.println("已完成该批次地址标准化,耗时" + stopWatch.getTotalTimeSeconds() / 60); + + for (List data : dataList) { + String address = ""; + if (data.size() == column) { + address = data.get(column - 1); + } else { + address = ""; + } + List addressRow = dataMap.get(address); + if (CollectionUtils.isNotEmpty(addressRow)) { + data.addAll(addressRow); + } + } + } catch (InterruptedException e) { + e.printStackTrace(); + } finally { + executorService.shutdown(); + } + + + while (true) { + try { + ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build(); + WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build(); + excelWriter.write(dataList, writeSheet); + excelWriter.finish(); + break; + } catch (Exception ex) { + ex.printStackTrace(); + System.out.println("五秒后重试"); + Thread.sleep(5000); + } + } + } + + private List> formatHead(String[] headArr) { + List> head = new ArrayList<>(); + for (String name: headArr) { + List temp = new ArrayList<>(); + temp.add(name); + head.add(temp); + } + return head; + } +}