江阴跑批
This commit is contained in:
+74
-59
@@ -4,6 +4,7 @@ import com.alibaba.excel.util.DateUtils;
|
|||||||
import com.alibaba.excel.util.StringUtils;
|
import com.alibaba.excel.util.StringUtils;
|
||||||
import com.alibaba.excel.write.metadata.WriteSheet;
|
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||||
import com.alibaba.fastjson.JSONObject;
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import dto.AddressResultDto;
|
||||||
import dto.FangJiaApiResponse;
|
import dto.FangJiaApiResponse;
|
||||||
import org.apache.commons.collections4.CollectionUtils;
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
import org.springframework.util.StopWatch;
|
import org.springframework.util.StopWatch;
|
||||||
@@ -12,6 +13,7 @@ import org.springframework.web.client.RestTemplate;
|
|||||||
import java.io.FileInputStream;
|
import java.io.FileInputStream;
|
||||||
import java.io.FileNotFoundException;
|
import java.io.FileNotFoundException;
|
||||||
import java.net.URLEncoder;
|
import java.net.URLEncoder;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
import java.util.*;
|
import java.util.*;
|
||||||
import java.util.concurrent.ConcurrentHashMap;
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
import java.util.concurrent.CountDownLatch;
|
import java.util.concurrent.CountDownLatch;
|
||||||
@@ -20,30 +22,45 @@ import java.util.concurrent.Executors;
|
|||||||
import java.util.concurrent.atomic.AtomicLong;
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
public class BatchTest {
|
public class BatchTest {
|
||||||
|
// IP地址
|
||||||
|
static final String IP = "192.168.0.100:33080";
|
||||||
|
//文件地址
|
||||||
|
static final String sourceFilePath = "E:\\goojia\\file\\excel文件\\江阴\\20240521-dzbzh.xlsx";
|
||||||
|
static final int column = 2;
|
||||||
|
|
||||||
|
//线程数
|
||||||
|
static final Integer nThreads = 30;
|
||||||
|
//涉农类型,1:对私,2:对公
|
||||||
|
static final String type = "1";
|
||||||
|
|
||||||
final int column = 2;
|
|
||||||
|
|
||||||
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
||||||
BatchTest test = new BatchTest();
|
VillageIdentifyTest test = new VillageIdentifyTest();
|
||||||
test.run(args[0], args[1], args[2], Integer.valueOf(args[3]));
|
test.run(IP, sourceFilePath, nThreads);
|
||||||
}
|
}
|
||||||
|
|
||||||
public void run(String ip, String file, String typeT, Integer nThreads) throws FileNotFoundException, InterruptedException {
|
public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException {
|
||||||
|
|
||||||
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
|
String formattedDate = sdf.format(new Date());
|
||||||
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
||||||
final String type = StringUtils.isEmpty(typeT) ? "1" : typeT;
|
|
||||||
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
||||||
AtomicLong count = new AtomicLong(0);
|
AtomicLong count = new AtomicLong(0);
|
||||||
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证", "经度", "纬度", "百度经度", "百度纬度", "提示"};
|
// String[] headArr = new String[]{
|
||||||
|
// "ID", "地址", "省份", "省份编码", "城市", "城市编码", "区域", "区域编码", "乡镇街道", "乡镇街道编码", "居委村委", "居委村委编码", "城乡分类编码",
|
||||||
|
// "城乡分类", "涉农标识", "地址验证", "是否县级市", "是否市辖建制镇", "是否开发区", "开发区名称", "开发区等级", "是否县政府"};
|
||||||
|
String[] headArr = new String[]{
|
||||||
|
"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否农户地址", "地址验证", "是否县级市", "是否市辖建制镇", "省国家级开发区名称", "开发区等级", "经度", "纬度", "百度经度", "百度纬度"};
|
||||||
List<List<String>> head = this.formatHead(headArr);
|
List<List<String>> head = this.formatHead(headArr);
|
||||||
// 默认两行数据
|
// 默认两行数据
|
||||||
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果" + DateUtils.format(new Date(), "yyyyMMddHHmm") + ".xlsx" ;
|
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx";
|
||||||
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
|
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
|
||||||
if (dataList.size() > 1000000) {
|
if (dataList.size() > 1000000) {
|
||||||
throw new RuntimeException("该批次数据量超过1000000条");
|
throw new RuntimeException("该批次数据量超过1000000条");
|
||||||
}
|
}
|
||||||
Set<String> set = new HashSet<>();
|
Set<String> set = new HashSet<>();
|
||||||
dataList.forEach(data -> {
|
dataList.forEach(data -> {
|
||||||
if (data.size() == column ) {
|
if (data.size() == column) {
|
||||||
set.add(data.get(column - 1));
|
set.add(data.get(column - 1));
|
||||||
} else {
|
} else {
|
||||||
// 存在为空情况,需保持一致
|
// 存在为空情况,需保持一致
|
||||||
@@ -53,7 +70,8 @@ public class BatchTest {
|
|||||||
});
|
});
|
||||||
|
|
||||||
StopWatch stopWatch = new StopWatch();
|
StopWatch stopWatch = new StopWatch();
|
||||||
stopWatch.start("总标准化耗时");
|
stopWatch.start("总耗时");
|
||||||
|
System.out.println("开始时间:" + formattedDate);
|
||||||
try {
|
try {
|
||||||
CountDownLatch latch = new CountDownLatch(set.size());
|
CountDownLatch latch = new CountDownLatch(set.size());
|
||||||
RestTemplate restTemplate = new RestTemplate();
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
@@ -61,67 +79,62 @@ public class BatchTest {
|
|||||||
final String address = addressTemp;
|
final String address = addressTemp;
|
||||||
executorService.submit(() -> {
|
executorService.submit(() -> {
|
||||||
try {
|
try {
|
||||||
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}&type={type}", String.class, address, type);
|
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}&type={type}&city=江阴市", String.class, address, type);
|
||||||
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
List<String> addressList = new ArrayList<>();
|
List<String> result = new ArrayList<>();
|
||||||
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
|
AddressResultDto villageIdentifyResultDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class);
|
||||||
addressList.add(structuredAddress.getProvince());
|
result.add(villageIdentifyResultDto.getProvince());
|
||||||
addressList.add(structuredAddress.getCity());
|
// result.add(villageIdentifyResultDto.getProvinceCode());
|
||||||
addressList.add(structuredAddress.getDistrict());
|
result.add(villageIdentifyResultDto.getCity());
|
||||||
addressList.add(structuredAddress.getTownship());
|
// result.add(villageIdentifyResultDto.getCityCode());
|
||||||
addressList.add(structuredAddress.getTownshipCode());
|
result.add(villageIdentifyResultDto.getDistrict());
|
||||||
addressList.add(structuredAddress.getCommunity());
|
// result.add(villageIdentifyResultDto.getDistrictCode());
|
||||||
addressList.add(structuredAddress.getFormatAddress());
|
result.add(villageIdentifyResultDto.getTownship());
|
||||||
addressList.add(structuredAddress.getIsAgriculture());
|
result.add(villageIdentifyResultDto.getTownshipCode());
|
||||||
addressList.add(structuredAddress.getAddrVerify());
|
result.add(villageIdentifyResultDto.getCommunity());
|
||||||
addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : "");
|
// result.add(villageIdentifyResultDto.getCommitteeCode());
|
||||||
addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : "");
|
|
||||||
addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : "");
|
// result.add(villageIdentifyResultDto.getTypeCode());
|
||||||
addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : "");
|
// result.add(villageIdentifyResultDto.getTypeName());
|
||||||
addressList.add(structuredAddress.getTips());
|
result.add(villageIdentifyResultDto.getFormatAddress());
|
||||||
dataMap.put(address, addressList);
|
result.add(villageIdentifyResultDto.getIsAgriculture());
|
||||||
|
result.add(villageIdentifyResultDto.getAddrVerify());
|
||||||
|
result.add(villageIdentifyResultDto.getCountyLevelCity());
|
||||||
|
result.add(villageIdentifyResultDto.getAdministrativeTowns());
|
||||||
|
// result.add(villageIdentifyResultDto.getDevDistrict());
|
||||||
|
result.add(villageIdentifyResultDto.getDevDistrictName());
|
||||||
|
result.add(villageIdentifyResultDto.getDevLevel());
|
||||||
|
if (villageIdentifyResultDto.getLat() != null &&
|
||||||
|
villageIdentifyResultDto.getLat() != null &&
|
||||||
|
villageIdentifyResultDto.getBdLat() != null &&
|
||||||
|
villageIdentifyResultDto.getBdLng() != null) {
|
||||||
|
result.add(villageIdentifyResultDto.getLng().toString());
|
||||||
|
result.add(villageIdentifyResultDto.getLat().toString());
|
||||||
|
result.add(villageIdentifyResultDto.getBdLng().toString());
|
||||||
|
result.add(villageIdentifyResultDto.getBdLat().toString());
|
||||||
|
}
|
||||||
|
// result.add(villageIdentifyResultDto.getCountyGov());
|
||||||
|
|
||||||
|
|
||||||
|
dataMap.put(address, result);
|
||||||
}
|
}
|
||||||
} catch (Exception ex) {
|
} catch (Exception ex) {
|
||||||
ex.printStackTrace();
|
ex.printStackTrace();
|
||||||
System.out.println("五秒后重试");
|
// System.out.println("10ms重试");
|
||||||
try {
|
try {
|
||||||
Thread.sleep(5000);
|
Thread.sleep(1);
|
||||||
|
System.out.println("地址" + address + "异常");
|
||||||
} catch (InterruptedException e) {
|
} catch (InterruptedException e) {
|
||||||
|
|
||||||
}
|
}
|
||||||
String response = restTemplate.getForObject("http://" + ip + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
|
|
||||||
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
|
||||||
try {
|
|
||||||
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
|
||||||
List<String> addressList = new ArrayList<>();
|
|
||||||
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
|
|
||||||
addressList.add(structuredAddress.getProvince());
|
|
||||||
addressList.add(structuredAddress.getCity());
|
|
||||||
addressList.add(structuredAddress.getDistrict());
|
|
||||||
addressList.add(structuredAddress.getTownship());
|
|
||||||
addressList.add(structuredAddress.getTownshipCode());
|
|
||||||
addressList.add(structuredAddress.getCommunity());
|
|
||||||
addressList.add(structuredAddress.getFormatAddress());
|
|
||||||
addressList.add(structuredAddress.getIsAgriculture());
|
|
||||||
addressList.add(structuredAddress.getAddrVerify());
|
|
||||||
addressList.add(structuredAddress.getLng() != null ? String.valueOf(structuredAddress.getLng()) : "");
|
|
||||||
addressList.add(structuredAddress.getLat() != null ? String.valueOf(structuredAddress.getLat()) : "");
|
|
||||||
addressList.add(structuredAddress.getBdLng() != null ? String.valueOf(structuredAddress.getBdLng()) : "");
|
|
||||||
addressList.add(structuredAddress.getBdLat() != null ? String.valueOf(structuredAddress.getBdLat()) : "");
|
|
||||||
addressList.add(structuredAddress.getTips());
|
|
||||||
dataMap.put(address, addressList);
|
|
||||||
}
|
|
||||||
} catch (Exception e) {
|
|
||||||
System.out.println("地址" + address + "异常");
|
|
||||||
}
|
|
||||||
} finally {
|
} finally {
|
||||||
latch.countDown();
|
latch.countDown();
|
||||||
count.incrementAndGet();
|
count.incrementAndGet();
|
||||||
synchronized (this) {
|
synchronized (this) {
|
||||||
if (count.longValue() % 1000 == 0) {
|
if (count.longValue() % 1000 == 0) {
|
||||||
stopWatch.stop();
|
stopWatch.stop();
|
||||||
System.out.println("已标准化处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() / 60);
|
System.out.println("已处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() + "秒");
|
||||||
stopWatch.start();
|
stopWatch.start();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -132,8 +145,8 @@ public class BatchTest {
|
|||||||
if (stopWatch.isRunning()) {
|
if (stopWatch.isRunning()) {
|
||||||
stopWatch.stop();
|
stopWatch.stop();
|
||||||
}
|
}
|
||||||
System.out.println("已完成该批次地址标准化,耗时" + stopWatch.getTotalTimeSeconds() / 60);
|
System.out.println("已完成该批次涉农识别,耗时" + stopWatch.getTotalTimeSeconds() / 60 + "分钟");
|
||||||
|
System.out.println("完成时间:" + sdf.format(new Date()));
|
||||||
for (List<String> data : dataList) {
|
for (List<String> data : dataList) {
|
||||||
String address = "";
|
String address = "";
|
||||||
if (data.size() == column) {
|
if (data.size() == column) {
|
||||||
@@ -156,7 +169,7 @@ public class BatchTest {
|
|||||||
while (true) {
|
while (true) {
|
||||||
try {
|
try {
|
||||||
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
|
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
|
||||||
WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build();
|
WriteSheet writeSheet = EasyExcel.writerSheet(0, "涉农结果").head(head).build();
|
||||||
excelWriter.write(dataList, writeSheet);
|
excelWriter.write(dataList, writeSheet);
|
||||||
excelWriter.finish();
|
excelWriter.finish();
|
||||||
break;
|
break;
|
||||||
@@ -166,11 +179,13 @@ public class BatchTest {
|
|||||||
Thread.sleep(5000);
|
Thread.sleep(5000);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
private List<List<String>> formatHead(String[] headArr) {
|
private List<List<String>> formatHead(String[] headArr) {
|
||||||
List<List<String>> head = new ArrayList<>();
|
List<List<String>> head = new ArrayList<>();
|
||||||
for (String name: headArr) {
|
for (String name : headArr) {
|
||||||
List<String> temp = new ArrayList<>();
|
List<String> temp = new ArrayList<>();
|
||||||
temp.add(name);
|
temp.add(name);
|
||||||
head.add(temp);
|
head.add(temp);
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ package dto;
|
|||||||
public class AddressResultDto {
|
public class AddressResultDto {
|
||||||
|
|
||||||
private String no;//序号
|
private String no;//序号
|
||||||
|
private String isAgriculture;
|
||||||
|
|
||||||
private String address;
|
private String address;
|
||||||
|
|
||||||
@@ -41,11 +42,67 @@ public class AddressResultDto {
|
|||||||
private String splitStreetNum;
|
private String splitStreetNum;
|
||||||
private String communitySource;//居委来源
|
private String communitySource;//居委来源
|
||||||
|
|
||||||
|
private String community;
|
||||||
|
|
||||||
|
public String getCommunity() {
|
||||||
|
return community;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommunity(String community) {
|
||||||
|
this.community = community;
|
||||||
|
}
|
||||||
|
|
||||||
private String township;
|
private String township;
|
||||||
private String townshipCode;
|
private String townshipCode;
|
||||||
private String formatAddress;
|
private String formatAddress;
|
||||||
private String addressAssessment;//地址验证
|
private String addressAssessment;//地址验证
|
||||||
|
|
||||||
|
private Double bdLng;
|
||||||
|
|
||||||
|
private Double bdLat;
|
||||||
|
private Double lng;
|
||||||
|
private Double lat;
|
||||||
|
|
||||||
|
public String getIsAgriculture() {
|
||||||
|
return isAgriculture;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setIsAgriculture(String isAgriculture) {
|
||||||
|
this.isAgriculture = isAgriculture;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Double getBdLng() {
|
||||||
|
return bdLng;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBdLng(Double bdLng) {
|
||||||
|
this.bdLng = bdLng;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Double getBdLat() {
|
||||||
|
return bdLat;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setBdLat(Double bdLat) {
|
||||||
|
this.bdLat = bdLat;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Double getLng() {
|
||||||
|
return lng;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLng(Double lng) {
|
||||||
|
this.lng = lng;
|
||||||
|
}
|
||||||
|
|
||||||
|
public Double getLat() {
|
||||||
|
return lat;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setLat(Double lat) {
|
||||||
|
this.lat = lat;
|
||||||
|
}
|
||||||
|
|
||||||
public String getName() {
|
public String getName() {
|
||||||
return name;
|
return name;
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user