From a9059bd5ae8a59f41d9b3e38a4efba7f9341cbeb Mon Sep 17 00:00:00 2001 From: mjt <13607600062@163.com> Date: Thu, 23 May 2024 13:14:51 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B1=9F=E9=98=B4=E8=B7=91=E6=89=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/BatchTest.java | 133 +++++++++++++++++++--------------- src/dto/AddressResultDto.java | 57 +++++++++++++++ 2 files changed, 131 insertions(+), 59 deletions(-) diff --git a/src/BatchTest.java b/src/BatchTest.java index b5c2819..f8cd616 100644 --- a/src/BatchTest.java +++ b/src/BatchTest.java @@ -4,6 +4,7 @@ import com.alibaba.excel.util.DateUtils; import com.alibaba.excel.util.StringUtils; import com.alibaba.excel.write.metadata.WriteSheet; import com.alibaba.fastjson.JSONObject; +import dto.AddressResultDto; import dto.FangJiaApiResponse; import org.apache.commons.collections4.CollectionUtils; import org.springframework.util.StopWatch; @@ -12,6 +13,7 @@ import org.springframework.web.client.RestTemplate; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.net.URLEncoder; +import java.text.SimpleDateFormat; import java.util.*; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -20,30 +22,45 @@ import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicLong; public class BatchTest { + // IP地址 + static final String IP = "192.168.0.100:33080"; + //文件地址 + static final String sourceFilePath = "E:\\goojia\\file\\excel文件\\江阴\\20240521-dzbzh.xlsx"; + static final int column = 2; + + //线程数 + static final Integer nThreads = 30; + //涉农类型,1:对私,2:对公 + static final String type = "1"; - final int column = 2; public static void main(String[] args) throws FileNotFoundException, InterruptedException { - BatchTest test = new BatchTest(); - test.run(args[0], args[1], args[2], Integer.valueOf(args[3])); + VillageIdentifyTest test = new VillageIdentifyTest(); + test.run(IP, sourceFilePath, nThreads); } - public void run(String ip, String file, String typeT, Integer nThreads) throws FileNotFoundException, InterruptedException { + public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException { + + SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); + String formattedDate = sdf.format(new Date()); ExecutorService executorService = Executors.newFixedThreadPool(nThreads); - final String type = StringUtils.isEmpty(typeT) ? "1" : typeT; Map> dataMap = new ConcurrentHashMap<>(); AtomicLong count = new AtomicLong(0); - String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证", "经度", "纬度", "百度经度", "百度纬度", "提示"}; +// String[] headArr = new String[]{ +// "ID", "地址", "省份", "省份编码", "城市", "城市编码", "区域", "区域编码", "乡镇街道", "乡镇街道编码", "居委村委", "居委村委编码", "城乡分类编码", +// "城乡分类", "涉农标识", "地址验证", "是否县级市", "是否市辖建制镇", "是否开发区", "开发区名称", "开发区等级", "是否县政府"}; + String[] headArr = new String[]{ + "ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否农户地址", "地址验证", "是否县级市", "是否市辖建制镇", "省国家级开发区名称", "开发区等级", "经度", "纬度", "百度经度", "百度纬度"}; List> head = this.formatHead(headArr); // 默认两行数据 - String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果" + DateUtils.format(new Date(), "yyyyMMddHHmm") + ".xlsx" ; + String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx"; List> dataList = ExcelUtils.read(new FileInputStream(file)); if (dataList.size() > 1000000) { throw new RuntimeException("该批次数据量超过1000000条"); } Set set = new HashSet<>(); dataList.forEach(data -> { - if (data.size() == column ) { + if (data.size() == column) { set.add(data.get(column - 1)); } else { // 存在为空情况,需保持一致 @@ -53,7 +70,8 @@ public class BatchTest { }); StopWatch stopWatch = new StopWatch(); - stopWatch.start("总标准化耗时"); + stopWatch.start("总耗时"); + System.out.println("开始时间:" + formattedDate); try { CountDownLatch latch = new CountDownLatch(set.size()); RestTemplate restTemplate = new RestTemplate(); @@ -61,67 +79,62 @@ public class BatchTest { final String address = addressTemp; executorService.submit(() -> { try { - String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}&type={type}", String.class, address, type); + String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}&type={type}&city=江阴市", String.class, address, type); FangJiaApiResponse apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class); if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) { - List addressList = new ArrayList<>(); - DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class); - addressList.add(structuredAddress.getProvince()); - addressList.add(structuredAddress.getCity()); - addressList.add(structuredAddress.getDistrict()); - addressList.add(structuredAddress.getTownship()); - addressList.add(structuredAddress.getTownshipCode()); - addressList.add(structuredAddress.getCommunity()); - addressList.add(structuredAddress.getFormatAddress()); - addressList.add(structuredAddress.getIsAgriculture()); - addressList.add(structuredAddress.getAddrVerify()); - addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : ""); - addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : ""); - addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : ""); - addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : ""); - addressList.add(structuredAddress.getTips()); - dataMap.put(address, addressList); + List result = new ArrayList<>(); + AddressResultDto villageIdentifyResultDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class); + result.add(villageIdentifyResultDto.getProvince()); +// result.add(villageIdentifyResultDto.getProvinceCode()); + result.add(villageIdentifyResultDto.getCity()); +// result.add(villageIdentifyResultDto.getCityCode()); + result.add(villageIdentifyResultDto.getDistrict()); +// result.add(villageIdentifyResultDto.getDistrictCode()); + result.add(villageIdentifyResultDto.getTownship()); + result.add(villageIdentifyResultDto.getTownshipCode()); + result.add(villageIdentifyResultDto.getCommunity()); +// result.add(villageIdentifyResultDto.getCommitteeCode()); + +// result.add(villageIdentifyResultDto.getTypeCode()); +// result.add(villageIdentifyResultDto.getTypeName()); + result.add(villageIdentifyResultDto.getFormatAddress()); + result.add(villageIdentifyResultDto.getIsAgriculture()); + result.add(villageIdentifyResultDto.getAddrVerify()); + result.add(villageIdentifyResultDto.getCountyLevelCity()); + result.add(villageIdentifyResultDto.getAdministrativeTowns()); +// result.add(villageIdentifyResultDto.getDevDistrict()); + result.add(villageIdentifyResultDto.getDevDistrictName()); + result.add(villageIdentifyResultDto.getDevLevel()); + if (villageIdentifyResultDto.getLat() != null && + villageIdentifyResultDto.getLat() != null && + villageIdentifyResultDto.getBdLat() != null && + villageIdentifyResultDto.getBdLng() != null) { + result.add(villageIdentifyResultDto.getLng().toString()); + result.add(villageIdentifyResultDto.getLat().toString()); + result.add(villageIdentifyResultDto.getBdLng().toString()); + result.add(villageIdentifyResultDto.getBdLat().toString()); + } +// result.add(villageIdentifyResultDto.getCountyGov()); + + + dataMap.put(address, result); } } catch (Exception ex) { ex.printStackTrace(); - System.out.println("五秒后重试"); +// System.out.println("10ms重试"); try { - Thread.sleep(5000); + Thread.sleep(1); + System.out.println("地址" + address + "异常"); } catch (InterruptedException e) { } - String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address); - FangJiaApiResponse apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class); - try { - if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) { - List addressList = new ArrayList<>(); - DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class); - addressList.add(structuredAddress.getProvince()); - addressList.add(structuredAddress.getCity()); - addressList.add(structuredAddress.getDistrict()); - addressList.add(structuredAddress.getTownship()); - addressList.add(structuredAddress.getTownshipCode()); - addressList.add(structuredAddress.getCommunity()); - addressList.add(structuredAddress.getFormatAddress()); - addressList.add(structuredAddress.getIsAgriculture()); - addressList.add(structuredAddress.getAddrVerify()); - addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : ""); - addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : ""); - addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : ""); - addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : ""); - addressList.add(structuredAddress.getTips()); - dataMap.put(address, addressList); - } - } catch (Exception e) { - System.out.println("地址" + address + "异常"); - } } finally { latch.countDown(); count.incrementAndGet(); synchronized (this) { if (count.longValue() % 1000 == 0) { stopWatch.stop(); - System.out.println("已标准化处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() / 60); + System.out.println("已处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() + "秒"); stopWatch.start(); } } @@ -132,8 +145,8 @@ public class BatchTest { if (stopWatch.isRunning()) { stopWatch.stop(); } - System.out.println("已完成该批次地址标准化,耗时" + stopWatch.getTotalTimeSeconds() / 60); - + System.out.println("已完成该批次涉农识别,耗时" + stopWatch.getTotalTimeSeconds() / 60 + "分钟"); + System.out.println("完成时间:" + sdf.format(new Date())); for (List data : dataList) { String address = ""; if (data.size() == column) { @@ -156,7 +169,7 @@ public class BatchTest { while (true) { try { ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build(); - WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build(); + WriteSheet writeSheet = EasyExcel.writerSheet(0, "涉农结果").head(head).build(); excelWriter.write(dataList, writeSheet); excelWriter.finish(); break; @@ -166,11 +179,13 @@ public class BatchTest { Thread.sleep(5000); } } + } + private List> formatHead(String[] headArr) { List> head = new ArrayList<>(); - for (String name: headArr) { + for (String name : headArr) { List temp = new ArrayList<>(); temp.add(name); head.add(temp); diff --git a/src/dto/AddressResultDto.java b/src/dto/AddressResultDto.java index 7934930..a379571 100644 --- a/src/dto/AddressResultDto.java +++ b/src/dto/AddressResultDto.java @@ -8,6 +8,7 @@ package dto; public class AddressResultDto { private String no;//序号 + private String isAgriculture; private String address; @@ -41,11 +42,67 @@ public class AddressResultDto { private String splitStreetNum; private String communitySource;//居委来源 + private String community; + + public String getCommunity() { + return community; + } + + public void setCommunity(String community) { + this.community = community; + } + private String township; private String townshipCode; private String formatAddress; private String addressAssessment;//地址验证 + private Double bdLng; + + private Double bdLat; + private Double lng; + private Double lat; + + public String getIsAgriculture() { + return isAgriculture; + } + + public void setIsAgriculture(String isAgriculture) { + this.isAgriculture = isAgriculture; + } + + public Double getBdLng() { + return bdLng; + } + + public void setBdLng(Double bdLng) { + this.bdLng = bdLng; + } + + public Double getBdLat() { + return bdLat; + } + + public void setBdLat(Double bdLat) { + this.bdLat = bdLat; + } + + public Double getLng() { + return lng; + } + + public void setLng(Double lng) { + this.lng = lng; + } + + public Double getLat() { + return lat; + } + + public void setLat(Double lat) { + this.lat = lat; + } + public String getName() { return name; }