涉农 联想 五级跑批
This commit is contained in:
@@ -0,0 +1,199 @@
|
|||||||
|
import com.alibaba.excel.EasyExcel;
|
||||||
|
import com.alibaba.excel.ExcelWriter;
|
||||||
|
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||||
|
import com.alibaba.fastjson.JSONArray;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import dto.AddressResultDto;
|
||||||
|
import dto.FangJiaApiResponse;
|
||||||
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
|
import org.springframework.util.StopWatch;
|
||||||
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
public class AddressAssociationTest {
|
||||||
|
|
||||||
|
static final String IP = "192.168.0.100:32080";
|
||||||
|
static final String sourceFilePath = "E:\\download\\关键字.xlsx";
|
||||||
|
static final int column = 2;
|
||||||
|
//涉农类型,1:对私,2:对公
|
||||||
|
static final Integer nThreads = 10;
|
||||||
|
static final String type = "2";
|
||||||
|
|
||||||
|
|
||||||
|
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
||||||
|
AddressAssociationTest test = new AddressAssociationTest();
|
||||||
|
test.run(IP, sourceFilePath, nThreads);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException {
|
||||||
|
|
||||||
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
|
String formattedDate = sdf.format(new Date());
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
||||||
|
Map<String, List<AddressResultDto>> dataMap = new ConcurrentHashMap<>();
|
||||||
|
AtomicLong count = new AtomicLong(0);
|
||||||
|
String[] headArr = new String[]{"编号","地址", "省份", "城市", "区域", "乡镇街道", "居委村委", "名称", "详细地址"};
|
||||||
|
List<List<String>> head = this.formatHead(headArr);
|
||||||
|
// 默认两行数据
|
||||||
|
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx";
|
||||||
|
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
|
||||||
|
List<List<String>> resultList = new ArrayList<>();
|
||||||
|
if (dataList.size() > 1000000) {
|
||||||
|
throw new RuntimeException("该批次数据量超过1000000条");
|
||||||
|
}
|
||||||
|
Set<String> set = new HashSet<>();
|
||||||
|
dataList.forEach(data -> {
|
||||||
|
if (data.size() == column) {
|
||||||
|
set.add(data.get(column - 1));
|
||||||
|
} else {
|
||||||
|
// 存在为空情况,需保持一致
|
||||||
|
set.add("");
|
||||||
|
data.add("");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
StopWatch stopWatch = new StopWatch();
|
||||||
|
stopWatch.start("总耗时");
|
||||||
|
System.out.println("开始时间:" + formattedDate);
|
||||||
|
try {
|
||||||
|
CountDownLatch latch = new CountDownLatch(set.size());
|
||||||
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
|
for (String addressTemp : set) {
|
||||||
|
final String keywords = addressTemp;
|
||||||
|
executorService.submit(() -> {
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/v2/addressAssociation?keywords={keywords}&offset=5", String.class, keywords);
|
||||||
|
FangJiaApiResponse<JSONArray> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
|
||||||
|
// 获取 "result" 字段对应的数组
|
||||||
|
JSONArray resultArray = apiResponse.getResult();
|
||||||
|
|
||||||
|
List<AddressResultDto> associationList = new ArrayList<AddressResultDto>();
|
||||||
|
// 遍历数组,并将每个元素添加到 List 中
|
||||||
|
for (int i = 0; i < resultArray.size(); i++) {
|
||||||
|
AddressResultDto association = resultArray.getObject(i, AddressResultDto.class);
|
||||||
|
associationList.add(association);
|
||||||
|
}
|
||||||
|
|
||||||
|
dataMap.put(keywords, associationList);
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
try {
|
||||||
|
Thread.sleep(5000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/v2/addressAssociation?keywords={keywords}&offset=5", String.class, keywords);
|
||||||
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
|
||||||
|
// 获取 "result" 字段对应的数组
|
||||||
|
JSONArray resultArray = apiResponse.getResult().getJSONArray("result");
|
||||||
|
|
||||||
|
List<AddressResultDto> associationList = new ArrayList<AddressResultDto>();
|
||||||
|
// 遍历数组,并将每个元素添加到 List 中
|
||||||
|
for (int i = 0; i < resultArray.size(); i++) {
|
||||||
|
AddressResultDto association = resultArray.getObject(i, AddressResultDto.class);
|
||||||
|
associationList.add(association);
|
||||||
|
}
|
||||||
|
|
||||||
|
dataMap.put(keywords, associationList);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println("关键词" + keywords + "异常");
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
latch.countDown();
|
||||||
|
count.incrementAndGet();
|
||||||
|
synchronized (this) {
|
||||||
|
if (count.longValue() % 1000 == 0) {
|
||||||
|
stopWatch.stop();
|
||||||
|
System.out.println("已处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() + "秒");
|
||||||
|
stopWatch.start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await();
|
||||||
|
if (stopWatch.isRunning()) {
|
||||||
|
stopWatch.stop();
|
||||||
|
}
|
||||||
|
System.out.println("已完成该批次联想,耗时" + stopWatch.getTotalTimeSeconds() / 60 + "分钟");
|
||||||
|
System.out.println("完成时间:" + sdf.format(new Date()));
|
||||||
|
|
||||||
|
for (List<String> data : dataList) {
|
||||||
|
String keyword = "";
|
||||||
|
if (data.size() == column) {
|
||||||
|
keyword = data.get(column - 1);
|
||||||
|
} else {
|
||||||
|
keyword = "";
|
||||||
|
}
|
||||||
|
List<AddressResultDto> addressResultDtoList = dataMap.get(keyword);
|
||||||
|
if (CollectionUtils.isNotEmpty(addressResultDtoList)) {
|
||||||
|
for (AddressResultDto addressResultDto : addressResultDtoList) {
|
||||||
|
List<String> addressRow = new ArrayList<>();
|
||||||
|
addressRow.add(data.get(column - 2));
|
||||||
|
addressRow.add(keyword);
|
||||||
|
addressRow.add(addressResultDto.getProvince());
|
||||||
|
addressRow.add(addressResultDto.getCity());
|
||||||
|
addressRow.add(addressResultDto.getDistrict());
|
||||||
|
addressRow.add(addressResultDto.getTownship());
|
||||||
|
addressRow.add(addressResultDto.getCommittee());
|
||||||
|
addressRow.add(addressResultDto.getName());
|
||||||
|
addressRow.add(addressResultDto.getAddress());
|
||||||
|
if (CollectionUtils.isNotEmpty(addressRow)) {
|
||||||
|
resultList.add(addressRow);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} finally {
|
||||||
|
executorService.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
|
||||||
|
WriteSheet writeSheet = EasyExcel.writerSheet(0, "地址联想结果").head(head).build();
|
||||||
|
excelWriter.write(resultList, writeSheet);
|
||||||
|
excelWriter.finish();
|
||||||
|
break;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
Thread.sleep(5000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private List<List<String>> formatHead(String[] headArr) {
|
||||||
|
List<List<String>> head = new ArrayList<>();
|
||||||
|
for (String name : headArr) {
|
||||||
|
List<String> temp = new ArrayList<>();
|
||||||
|
temp.add(name);
|
||||||
|
head.add(temp);
|
||||||
|
}
|
||||||
|
return head;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,190 @@
|
|||||||
|
import com.alibaba.excel.EasyExcel;
|
||||||
|
import com.alibaba.excel.ExcelWriter;
|
||||||
|
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import dto.AddressResultDto;
|
||||||
|
import dto.FangJiaApiResponse;
|
||||||
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
|
import org.springframework.util.StopWatch;
|
||||||
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
public class FiveLevelStructure {
|
||||||
|
|
||||||
|
static final String IP = "192.168.0.100:32080";
|
||||||
|
static final String sourceFilePath = "E:\\download\\跑批测试.xlsx";
|
||||||
|
static final int column = 2;
|
||||||
|
static final Integer nThreads = 10;
|
||||||
|
|
||||||
|
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
||||||
|
FiveLevelStructure test = new FiveLevelStructure();
|
||||||
|
test.run(IP, sourceFilePath, nThreads);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException {
|
||||||
|
|
||||||
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
|
String formattedDate = sdf.format(new Date());
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
||||||
|
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
||||||
|
AtomicLong count = new AtomicLong(0);
|
||||||
|
String[] headArr = new String[]{
|
||||||
|
"编号", "地址", "省份", "城市", "区域", "乡镇街道", "地址", "标准化地址", "地址验证信息"};
|
||||||
|
List<List<String>> head = this.formatHead(headArr);
|
||||||
|
// 默认两行数据
|
||||||
|
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx";
|
||||||
|
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
|
||||||
|
if (dataList.size() > 1000000) {
|
||||||
|
throw new RuntimeException("该批次数据量超过1000000条");
|
||||||
|
}
|
||||||
|
Set<String> set = new HashSet<>();
|
||||||
|
dataList.forEach(data -> {
|
||||||
|
if (data.size() == column) {
|
||||||
|
set.add(data.get(column - 1));
|
||||||
|
} else {
|
||||||
|
// 存在为空情况,需保持一致
|
||||||
|
set.add("");
|
||||||
|
data.add("");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
StopWatch stopWatch = new StopWatch();
|
||||||
|
stopWatch.start("总耗时");
|
||||||
|
System.out.println("开始时间:" + formattedDate);
|
||||||
|
try {
|
||||||
|
CountDownLatch latch = new CountDownLatch(set.size());
|
||||||
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
|
for (String addressTemp : set) {
|
||||||
|
final String address = addressTemp;
|
||||||
|
executorService.submit(() -> {
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/v3/structure?address={address}&showCode=1&addressAssessment=true&complete=true", String.class, address);
|
||||||
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
AddressResultDto fiveLevelStructureDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class);
|
||||||
|
result.add(fiveLevelStructureDto.getProvince());
|
||||||
|
// result.add(fiveLevelStructureDto.getProvinceCode());
|
||||||
|
result.add(fiveLevelStructureDto.getCity());
|
||||||
|
// result.add(fiveLevelStructureDto.getCityCode());
|
||||||
|
result.add(fiveLevelStructureDto.getDistrict());
|
||||||
|
// result.add(fiveLevelStructureDto.getDistrictCode());
|
||||||
|
result.add(fiveLevelStructureDto.getTownship());
|
||||||
|
// result.add(fiveLevelStructureDto.getTownshipCode());
|
||||||
|
|
||||||
|
result.add(fiveLevelStructureDto.getAddress());
|
||||||
|
result.add(fiveLevelStructureDto.getFormatAddress());
|
||||||
|
|
||||||
|
String str = fiveLevelStructureDto.getAddressAssessment().split(",")[0].toString().split(":")[1].toString().replaceAll("\"", "");
|
||||||
|
|
||||||
|
result.add(str);
|
||||||
|
dataMap.put(address, result);
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
try {
|
||||||
|
Thread.sleep(5000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/v3/structure?address={address}&showCode=1&addressAssessment=true&complete=true", String.class, address);
|
||||||
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
AddressResultDto fiveLevelStructureDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class);
|
||||||
|
result.add(fiveLevelStructureDto.getProvince());
|
||||||
|
// result.add(fiveLevelStructureDto.getProvinceCode());
|
||||||
|
result.add(fiveLevelStructureDto.getCity());
|
||||||
|
// result.add(fiveLevelStructureDto.getCityCode());
|
||||||
|
result.add(fiveLevelStructureDto.getDistrict());
|
||||||
|
// result.add(fiveLevelStructureDto.getDistrictCode());
|
||||||
|
result.add(fiveLevelStructureDto.getTownship());
|
||||||
|
// result.add(fiveLevelStructureDto.getTownshipCode());
|
||||||
|
|
||||||
|
result.add(fiveLevelStructureDto.getAddress());
|
||||||
|
result.add(fiveLevelStructureDto.getFormatAddress());
|
||||||
|
|
||||||
|
String str = fiveLevelStructureDto.getAddressAssessment().split(",")[0].toString().split(":")[1].toString().replaceAll("\"", "");
|
||||||
|
|
||||||
|
result.add(str);
|
||||||
|
dataMap.put(address, result);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println("地址" + address + "异常");
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
latch.countDown();
|
||||||
|
count.incrementAndGet();
|
||||||
|
synchronized (this) {
|
||||||
|
if (count.longValue() % 1000 == 0) {
|
||||||
|
stopWatch.stop();
|
||||||
|
System.out.println("已处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() + "秒");
|
||||||
|
stopWatch.start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await();
|
||||||
|
if (stopWatch.isRunning()) {
|
||||||
|
stopWatch.stop();
|
||||||
|
}
|
||||||
|
System.out.println("已完成该批次五级标准化识别,耗时" + stopWatch.getTotalTimeSeconds() / 60 + "分钟");
|
||||||
|
System.out.println("完成时间:" + sdf.format(new Date()));
|
||||||
|
for (List<String> data : dataList) {
|
||||||
|
String address = "";
|
||||||
|
if (data.size() == column) {
|
||||||
|
address = data.get(column - 1);
|
||||||
|
} else {
|
||||||
|
address = "";
|
||||||
|
}
|
||||||
|
List<String> addressRow = dataMap.get(address);
|
||||||
|
if (CollectionUtils.isNotEmpty(addressRow)) {
|
||||||
|
data.addAll(addressRow);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} finally {
|
||||||
|
executorService.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
|
||||||
|
WriteSheet writeSheet = EasyExcel.writerSheet(0, "五级标准化结果").head(head).build();
|
||||||
|
excelWriter.write(dataList, writeSheet);
|
||||||
|
excelWriter.finish();
|
||||||
|
break;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
Thread.sleep(5000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private List<List<String>> formatHead(String[] headArr) {
|
||||||
|
List<List<String>> head = new ArrayList<>();
|
||||||
|
for (String name : headArr) {
|
||||||
|
List<String> temp = new ArrayList<>();
|
||||||
|
temp.add(name);
|
||||||
|
head.add(temp);
|
||||||
|
}
|
||||||
|
return head;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
import com.alibaba.excel.EasyExcel;
|
||||||
|
import com.alibaba.excel.ExcelWriter;
|
||||||
|
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||||
|
import com.alibaba.fastjson.JSONObject;
|
||||||
|
import dto.AddressResultDto;
|
||||||
|
import dto.FangJiaApiResponse;
|
||||||
|
import org.apache.commons.collections4.CollectionUtils;
|
||||||
|
import org.springframework.util.StopWatch;
|
||||||
|
import org.springframework.web.client.RestTemplate;
|
||||||
|
|
||||||
|
import java.io.FileInputStream;
|
||||||
|
import java.io.FileNotFoundException;
|
||||||
|
import java.text.SimpleDateFormat;
|
||||||
|
import java.util.*;
|
||||||
|
import java.util.concurrent.ConcurrentHashMap;
|
||||||
|
import java.util.concurrent.CountDownLatch;
|
||||||
|
import java.util.concurrent.ExecutorService;
|
||||||
|
import java.util.concurrent.Executors;
|
||||||
|
import java.util.concurrent.atomic.AtomicLong;
|
||||||
|
|
||||||
|
public class VillageIdentifyTest {
|
||||||
|
|
||||||
|
static final String IP = "192.168.0.100:32080";
|
||||||
|
static final String sourceFilePath = "E:\\download\\跑批测试.xlsx";
|
||||||
|
static final int column = 2;
|
||||||
|
//涉农类型,1:对私,2:对公
|
||||||
|
static final Integer nThreads = 10;
|
||||||
|
static final String type = "2";
|
||||||
|
|
||||||
|
|
||||||
|
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
||||||
|
VillageIdentifyTest test = new VillageIdentifyTest();
|
||||||
|
test.run(IP, sourceFilePath, nThreads);
|
||||||
|
}
|
||||||
|
|
||||||
|
public void run(String ip, String file, Integer nThreads) throws FileNotFoundException, InterruptedException {
|
||||||
|
|
||||||
|
SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
|
||||||
|
String formattedDate = sdf.format(new Date());
|
||||||
|
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
||||||
|
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
||||||
|
AtomicLong count = new AtomicLong(0);
|
||||||
|
String[] headArr = new String[]{
|
||||||
|
"编号", "地址", "省份", "省份编码", "城市", "城市编码", "区域", "区域编码", "乡镇街道", "乡镇街道编码", "居委村委", "居委村委编码", "城乡分类编码",
|
||||||
|
"城乡分类", "涉农标识", "地址验证", "是否县级市", "是否市辖建制镇", "是否开发区", "开发区名称", "开发区等级", "是否县政府"};
|
||||||
|
List<List<String>> head = this.formatHead(headArr);
|
||||||
|
// 默认两行数据
|
||||||
|
String aimFilePath = file.substring(0, file.lastIndexOf(".xlsx")) + "结果.xlsx";
|
||||||
|
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(file));
|
||||||
|
if (dataList.size() > 1000000) {
|
||||||
|
throw new RuntimeException("该批次数据量超过1000000条");
|
||||||
|
}
|
||||||
|
Set<String> set = new HashSet<>();
|
||||||
|
dataList.forEach(data -> {
|
||||||
|
if (data.size() == column) {
|
||||||
|
set.add(data.get(column - 1));
|
||||||
|
} else {
|
||||||
|
// 存在为空情况,需保持一致
|
||||||
|
set.add("");
|
||||||
|
data.add("");
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
StopWatch stopWatch = new StopWatch();
|
||||||
|
stopWatch.start("总耗时");
|
||||||
|
System.out.println("开始时间:"+ formattedDate);
|
||||||
|
try {
|
||||||
|
CountDownLatch latch = new CountDownLatch(set.size());
|
||||||
|
RestTemplate restTemplate = new RestTemplate();
|
||||||
|
for (String addressTemp : set) {
|
||||||
|
final String address = addressTemp;
|
||||||
|
executorService.submit(() -> {
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/villageIdentify?address={address}&type={type}&showFullValue=2", String.class, address, type);
|
||||||
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
AddressResultDto villageIdentifyResultDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class);
|
||||||
|
result.add(villageIdentifyResultDto.getProvince());
|
||||||
|
result.add(villageIdentifyResultDto.getProvinceCode());
|
||||||
|
result.add(villageIdentifyResultDto.getCity());
|
||||||
|
result.add(villageIdentifyResultDto.getCityCode());
|
||||||
|
result.add(villageIdentifyResultDto.getDistrict());
|
||||||
|
result.add(villageIdentifyResultDto.getDistrictCode());
|
||||||
|
result.add(villageIdentifyResultDto.getTown());
|
||||||
|
result.add(villageIdentifyResultDto.getTownCode());
|
||||||
|
result.add(villageIdentifyResultDto.getCommittee());
|
||||||
|
result.add(villageIdentifyResultDto.getCommitteeCode());
|
||||||
|
|
||||||
|
result.add(villageIdentifyResultDto.getTypeCode());
|
||||||
|
result.add(villageIdentifyResultDto.getTypeName());
|
||||||
|
result.add(villageIdentifyResultDto.getVerifyFlag());
|
||||||
|
result.add(villageIdentifyResultDto.getAddrVerify());
|
||||||
|
result.add(villageIdentifyResultDto.getCountyLevelCity());
|
||||||
|
result.add(villageIdentifyResultDto.getAdministrativeTowns());
|
||||||
|
result.add(villageIdentifyResultDto.getDevDistrict());
|
||||||
|
result.add(villageIdentifyResultDto.getDevDistrictName());
|
||||||
|
result.add(villageIdentifyResultDto.getDevLevel());
|
||||||
|
result.add(villageIdentifyResultDto.getCountyGov());
|
||||||
|
|
||||||
|
dataMap.put(address, result);
|
||||||
|
}
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
try {
|
||||||
|
Thread.sleep(5000);
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
String response = restTemplate.getForObject("http://" + IP + "FangJiaAddressService/villageIdentify?address={address}&type={type}", String.class, address, type);
|
||||||
|
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||||
|
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||||
|
List<String> result = new ArrayList<>();
|
||||||
|
AddressResultDto villageIdentifyResultDto = JSONObject.toJavaObject(apiResponse.getResult(), AddressResultDto.class);
|
||||||
|
result.add(villageIdentifyResultDto.getProvince());
|
||||||
|
result.add(villageIdentifyResultDto.getProvinceCode());
|
||||||
|
result.add(villageIdentifyResultDto.getCity());
|
||||||
|
result.add(villageIdentifyResultDto.getCityCode());
|
||||||
|
result.add(villageIdentifyResultDto.getDistrict());
|
||||||
|
result.add(villageIdentifyResultDto.getDistrictCode());
|
||||||
|
result.add(villageIdentifyResultDto.getTown());
|
||||||
|
result.add(villageIdentifyResultDto.getTownCode());
|
||||||
|
result.add(villageIdentifyResultDto.getCommittee());
|
||||||
|
result.add(villageIdentifyResultDto.getCommitteeCode());
|
||||||
|
|
||||||
|
result.add(villageIdentifyResultDto.getTypeCode());
|
||||||
|
result.add(villageIdentifyResultDto.getTypeName());
|
||||||
|
result.add(villageIdentifyResultDto.getVerifyFlag());
|
||||||
|
result.add(villageIdentifyResultDto.getAddrVerify());
|
||||||
|
result.add(villageIdentifyResultDto.getCountyLevelCity());
|
||||||
|
result.add(villageIdentifyResultDto.getAdministrativeTowns());
|
||||||
|
result.add(villageIdentifyResultDto.getDevDistrict());
|
||||||
|
result.add(villageIdentifyResultDto.getDevDistrictName());
|
||||||
|
result.add(villageIdentifyResultDto.getDevLevel());
|
||||||
|
result.add(villageIdentifyResultDto.getCountyGov());
|
||||||
|
|
||||||
|
dataMap.put(address, result);
|
||||||
|
}
|
||||||
|
} catch (Exception e) {
|
||||||
|
System.out.println("地址" + address + "异常");
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
latch.countDown();
|
||||||
|
count.incrementAndGet();
|
||||||
|
synchronized (this) {
|
||||||
|
if (count.longValue() % 1000 == 0) {
|
||||||
|
stopWatch.stop();
|
||||||
|
System.out.println("已处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds()+"秒");
|
||||||
|
stopWatch.start();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
|
latch.await();
|
||||||
|
if (stopWatch.isRunning()) {
|
||||||
|
stopWatch.stop();
|
||||||
|
}
|
||||||
|
System.out.println("已完成该批次涉农识别,耗时" + stopWatch.getTotalTimeSeconds()/60+"分钟");
|
||||||
|
System.out.println("完成时间:" + sdf.format(new Date()));
|
||||||
|
for (List<String> data : dataList) {
|
||||||
|
String address = "";
|
||||||
|
if (data.size() == column) {
|
||||||
|
address = data.get(column - 1);
|
||||||
|
} else {
|
||||||
|
address = "";
|
||||||
|
}
|
||||||
|
List<String> addressRow = dataMap.get(address);
|
||||||
|
if (CollectionUtils.isNotEmpty(addressRow)) {
|
||||||
|
data.addAll(addressRow);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (InterruptedException e) {
|
||||||
|
e.printStackTrace();
|
||||||
|
} finally {
|
||||||
|
executorService.shutdown();
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
while (true) {
|
||||||
|
try {
|
||||||
|
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
|
||||||
|
WriteSheet writeSheet = EasyExcel.writerSheet(0, "涉农结果").head(head).build();
|
||||||
|
excelWriter.write(dataList, writeSheet);
|
||||||
|
excelWriter.finish();
|
||||||
|
break;
|
||||||
|
} catch (Exception ex) {
|
||||||
|
ex.printStackTrace();
|
||||||
|
System.out.println("五秒后重试");
|
||||||
|
Thread.sleep(5000);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
private List<List<String>> formatHead(String[] headArr) {
|
||||||
|
List<List<String>> head = new ArrayList<>();
|
||||||
|
for (String name : headArr) {
|
||||||
|
List<String> temp = new ArrayList<>();
|
||||||
|
temp.add(name);
|
||||||
|
head.add(temp);
|
||||||
|
}
|
||||||
|
return head;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
package dto;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 涉农识别结果结果
|
||||||
|
*/
|
||||||
|
public class AddressResultDto {
|
||||||
|
|
||||||
|
private String no;//序号
|
||||||
|
|
||||||
|
private String address;
|
||||||
|
|
||||||
|
private String province;
|
||||||
|
private String provinceCode;
|
||||||
|
private String city;
|
||||||
|
private String cityCode;
|
||||||
|
private String district;
|
||||||
|
private String districtCode;
|
||||||
|
private String town;
|
||||||
|
private String townCode;
|
||||||
|
private String committee;
|
||||||
|
private String committeeCode;
|
||||||
|
private String typeCode; //城乡分类
|
||||||
|
private String typeName; //城乡代码
|
||||||
|
private String verifyFlag;//涉农标识
|
||||||
|
private String addrVerify;//地址验证
|
||||||
|
private String countyLevelCity;//是否为县级市
|
||||||
|
private String administrativeTowns;//是否市辖建制镇
|
||||||
|
private String countyGov;//是否为县政府
|
||||||
|
private String devDistrict;//是否为开发区
|
||||||
|
private String devDistrictName;//开发区
|
||||||
|
private String devLevel;//开发区等级
|
||||||
|
private String name;
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
private String splitCommittee;
|
||||||
|
private String splitStreet;
|
||||||
|
private String poiName;
|
||||||
|
private String splitStreetNum;
|
||||||
|
private String communitySource;//居委来源
|
||||||
|
|
||||||
|
private String township;
|
||||||
|
private String townshipCode;
|
||||||
|
private String formatAddress;
|
||||||
|
private String addressAssessment;//地址验证
|
||||||
|
|
||||||
|
public String getName() {
|
||||||
|
return name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setName(String name) {
|
||||||
|
this.name = name;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTownship() {
|
||||||
|
return township;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTownship(String township) {
|
||||||
|
this.township = township;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTownshipCode() {
|
||||||
|
return townshipCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTownshipCode(String townshipCode) {
|
||||||
|
this.townshipCode = townshipCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getFormatAddress() {
|
||||||
|
return formatAddress;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setFormatAddress(String formatAddress) {
|
||||||
|
this.formatAddress = formatAddress;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAddressAssessment() {
|
||||||
|
return addressAssessment;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAddressAssessment(String addressAssessment) {
|
||||||
|
this.addressAssessment = addressAssessment;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getNo() {
|
||||||
|
return no;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setNo(String no) {
|
||||||
|
this.no = no;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAddress() {
|
||||||
|
return address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAddress(String address) {
|
||||||
|
this.address = address;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getProvince() {
|
||||||
|
return province;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProvince(String province) {
|
||||||
|
this.province = province;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getProvinceCode() {
|
||||||
|
return provinceCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setProvinceCode(String provinceCode) {
|
||||||
|
this.provinceCode = provinceCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCity() {
|
||||||
|
return city;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCity(String city) {
|
||||||
|
this.city = city;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCityCode() {
|
||||||
|
return cityCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCityCode(String cityCode) {
|
||||||
|
this.cityCode = cityCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDistrict() {
|
||||||
|
return district;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDistrict(String district) {
|
||||||
|
this.district = district;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDistrictCode() {
|
||||||
|
return districtCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDistrictCode(String districtCode) {
|
||||||
|
this.districtCode = districtCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTown() {
|
||||||
|
return town;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTown(String town) {
|
||||||
|
this.town = town;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTownCode() {
|
||||||
|
return townCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTownCode(String townCode) {
|
||||||
|
this.townCode = townCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCommittee() {
|
||||||
|
return committee;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommittee(String committee) {
|
||||||
|
this.committee = committee;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCommitteeCode() {
|
||||||
|
return committeeCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommitteeCode(String committeeCode) {
|
||||||
|
this.committeeCode = committeeCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTypeCode() {
|
||||||
|
return typeCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTypeCode(String typeCode) {
|
||||||
|
this.typeCode = typeCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getTypeName() {
|
||||||
|
return typeName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setTypeName(String typeName) {
|
||||||
|
this.typeName = typeName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getVerifyFlag() {
|
||||||
|
return verifyFlag;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setVerifyFlag(String verifyFlag) {
|
||||||
|
this.verifyFlag = verifyFlag;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAddrVerify() {
|
||||||
|
return addrVerify;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAddrVerify(String addrVerify) {
|
||||||
|
this.addrVerify = addrVerify;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCountyLevelCity() {
|
||||||
|
return countyLevelCity;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCountyLevelCity(String countyLevelCity) {
|
||||||
|
this.countyLevelCity = countyLevelCity;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getAdministrativeTowns() {
|
||||||
|
return administrativeTowns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setAdministrativeTowns(String administrativeTowns) {
|
||||||
|
this.administrativeTowns = administrativeTowns;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCountyGov() {
|
||||||
|
return countyGov;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCountyGov(String countyGov) {
|
||||||
|
this.countyGov = countyGov;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDevDistrict() {
|
||||||
|
return devDistrict;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDevDistrict(String devDistrict) {
|
||||||
|
this.devDistrict = devDistrict;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDevDistrictName() {
|
||||||
|
return devDistrictName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDevDistrictName(String devDistrictName) {
|
||||||
|
this.devDistrictName = devDistrictName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getDevLevel() {
|
||||||
|
return devLevel;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setDevLevel(String devLevel) {
|
||||||
|
this.devLevel = devLevel;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSplitCommittee() {
|
||||||
|
return splitCommittee;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSplitCommittee(String splitCommittee) {
|
||||||
|
this.splitCommittee = splitCommittee;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSplitStreet() {
|
||||||
|
return splitStreet;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSplitStreet(String splitStreet) {
|
||||||
|
this.splitStreet = splitStreet;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getPoiName() {
|
||||||
|
return poiName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setPoiName(String poiName) {
|
||||||
|
this.poiName = poiName;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getSplitStreetNum() {
|
||||||
|
return splitStreetNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setSplitStreetNum(String splitStreetNum) {
|
||||||
|
this.splitStreetNum = splitStreetNum;
|
||||||
|
}
|
||||||
|
|
||||||
|
public String getCommunitySource() {
|
||||||
|
return communitySource;
|
||||||
|
}
|
||||||
|
|
||||||
|
public void setCommunitySource(String communitySource) {
|
||||||
|
this.communitySource = communitySource;
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user