新增地址补全调用
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
import com.alibaba.excel.EasyExcel;
|
||||
import com.alibaba.excel.ExcelWriter;
|
||||
import com.alibaba.excel.write.metadata.WriteSheet;
|
||||
import com.alibaba.fastjson.JSONObject;
|
||||
import org.apache.commons.collections4.CollectionUtils;
|
||||
import org.springframework.util.StopWatch;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileNotFoundException;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
public class DivisionBatchTest {
|
||||
|
||||
static final String IP = "192.168.0.19:31080";
|
||||
final int column = 2;
|
||||
final Integer nThreads = 20;
|
||||
|
||||
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
|
||||
DivisionBatchTest test = new DivisionBatchTest();
|
||||
String[] cities = new String[]{"北京", "青岛", "上海", "潍坊", "烟台"};
|
||||
for (String city : cities) {
|
||||
final String sourceFilePath = "D:\\goojia\\中信测试\\" + city + "地址标准化测试.xlsx";
|
||||
final String aimFilePath = "D:\\goojia\\中信测试\\" + city + "地址标准化测试结果.xlsx";
|
||||
test.run(city, sourceFilePath, aimFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
public void run(String city, String sourceFilePath, String aimFilePath) throws FileNotFoundException, InterruptedException {
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
|
||||
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
|
||||
AtomicLong count = new AtomicLong(0);
|
||||
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "开发区", "乡镇街道", "街道编码", "居委村委", "道路", "子路",
|
||||
"路号", "村组", "名称", "楼栋", "单元", "楼层", "室号", "地址验证"};
|
||||
List<List<String>> head = this.formatHead(headArr);
|
||||
// 默认两行数据
|
||||
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(sourceFilePath));
|
||||
if (dataList.size() > 100000) {
|
||||
throw new RuntimeException("该批次数据量超过100000条");
|
||||
}
|
||||
Set<String> set = new HashSet<>();
|
||||
dataList.forEach(data -> {
|
||||
if (data.size() == column ) {
|
||||
set.add(data.get(column - 1));
|
||||
} else {
|
||||
// 存在为空情况,需保持一致
|
||||
set.add("");
|
||||
data.add("");
|
||||
}
|
||||
});
|
||||
|
||||
StopWatch stopWatch = new StopWatch();
|
||||
stopWatch.start("总标准化耗时");
|
||||
try {
|
||||
CountDownLatch latch = new CountDownLatch(set.size());
|
||||
RestTemplate restTemplate = new RestTemplate();
|
||||
for (String addressTemp : set) {
|
||||
final String address = addressTemp;
|
||||
executorService.submit(() -> {
|
||||
try {
|
||||
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/divisionStructure?" +
|
||||
"city={city}&address={address}", String.class, city, address);
|
||||
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||
List<String> addressList = new ArrayList<>();
|
||||
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
|
||||
addressList.add(structuredAddress.getProvince());
|
||||
addressList.add(structuredAddress.getCity());
|
||||
addressList.add(structuredAddress.getDistrict());
|
||||
addressList.add(structuredAddress.getDev());
|
||||
addressList.add(structuredAddress.getTownship());
|
||||
addressList.add(structuredAddress.getTownshipCode());
|
||||
addressList.add(structuredAddress.getCommunity());
|
||||
addressList.add(structuredAddress.getStreet());
|
||||
addressList.add(structuredAddress.getSubStreet());
|
||||
addressList.add(structuredAddress.getStreetNumber());
|
||||
addressList.add(structuredAddress.getGroup());
|
||||
addressList.add(structuredAddress.getName());
|
||||
addressList.add(structuredAddress.getBuildingNumber());
|
||||
addressList.add(structuredAddress.getUnitNumber());
|
||||
addressList.add(structuredAddress.getFloor());
|
||||
addressList.add(structuredAddress.getRoomNumber());
|
||||
addressList.add(structuredAddress.getAddrVerify());
|
||||
dataMap.put(address, addressList);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
System.out.println("五秒后重试");
|
||||
try {
|
||||
Thread.sleep(5000);
|
||||
} catch (InterruptedException e) {
|
||||
|
||||
}
|
||||
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?city={city}&address={address}", String.class, city, address);
|
||||
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
|
||||
try {
|
||||
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
|
||||
List<String> addressList = new ArrayList<>();
|
||||
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
|
||||
addressList.add(structuredAddress.getProvince());
|
||||
addressList.add(structuredAddress.getCity());
|
||||
addressList.add(structuredAddress.getDistrict());
|
||||
addressList.add(structuredAddress.getTownship());
|
||||
addressList.add(structuredAddress.getTownshipCode());
|
||||
addressList.add(structuredAddress.getCommunity());
|
||||
addressList.add(structuredAddress.getFormatAddress());
|
||||
addressList.add(structuredAddress.getAddrVerify());
|
||||
dataMap.put(address, addressList);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
System.out.println("give up");
|
||||
}
|
||||
} finally {
|
||||
latch.countDown();
|
||||
count.incrementAndGet();
|
||||
synchronized (this) {
|
||||
if (count.longValue() % 1000 == 0) {
|
||||
stopWatch.stop();
|
||||
System.out.println("已标准化处理" + count.longValue() + "条地址, 耗时:" + stopWatch.getTotalTimeSeconds() / 60);
|
||||
stopWatch.start();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await();
|
||||
if (stopWatch.isRunning()) {
|
||||
stopWatch.stop();
|
||||
}
|
||||
System.out.println("已完成该批次地址标准化,耗时" + stopWatch.getTotalTimeSeconds() / 60);
|
||||
|
||||
for (List<String> data : dataList) {
|
||||
String address = "";
|
||||
if (data.size() == column) {
|
||||
address = data.get(column - 1);
|
||||
} else {
|
||||
address = "";
|
||||
}
|
||||
List<String> addressRow = dataMap.get(address);
|
||||
if (CollectionUtils.isNotEmpty(addressRow)) {
|
||||
data.addAll(addressRow);
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
e.printStackTrace();
|
||||
} finally {
|
||||
executorService.shutdown();
|
||||
}
|
||||
|
||||
|
||||
while (true) {
|
||||
try {
|
||||
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
|
||||
WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build();
|
||||
excelWriter.write(dataList, writeSheet);
|
||||
excelWriter.finish();
|
||||
break;
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace();
|
||||
System.out.println("五秒后重试");
|
||||
Thread.sleep(5000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<List<String>> formatHead(String[] headArr) {
|
||||
List<List<String>> head = new ArrayList<>();
|
||||
for (String name: headArr) {
|
||||
List<String> temp = new ArrayList<>();
|
||||
temp.add(name);
|
||||
head.add(temp);
|
||||
}
|
||||
return head;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user