本地化部署测试初始化

This commit is contained in:
沈健
2021-11-09 11:46:43 +08:00
commit d97264166a
32 changed files with 782 additions and 0 deletions
+164
View File
@@ -0,0 +1,164 @@
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
public class BatchTest {
static final String IP = "192.168.0.19:39080";
final String sourceFilePath = "D:\\goojia\\20210715090234.xlsx";
final String aimFilePath = "D:\\goojia\\20210715090234最新结果.xlsx";
final int column = 2;
final Integer nThreads = 20;
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
BatchTest test = new BatchTest();
test.run();
}
public void run() throws FileNotFoundException, InterruptedException {
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
AtomicLong count = new AtomicLong(0);
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证"};
List<List<String>> head = this.formatHead(headArr);
// 默认两行数据
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(sourceFilePath));
if (dataList.size() > 100000) {
throw new RuntimeException("该批次数据量超过100000条");
}
Set<String> set = new HashSet<>();
dataList.forEach(data -> {
if (data.size() == column ) {
set.add(data.get(column - 1));
} else {
// 存在为空情况,需保持一致
set.add("");
data.add("");
}
});
try {
CountDownLatch latch = new CountDownLatch(set.size());
RestTemplate restTemplate = new RestTemplate();
for (String addressTemp : set) {
final String address = addressTemp;
executorService.submit(() -> {
try {
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> addressList = new ArrayList<>();
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
addressList.add(structuredAddress.getProvince());
addressList.add(structuredAddress.getCity());
addressList.add(structuredAddress.getDistrict());
addressList.add(structuredAddress.getTownship());
addressList.add(structuredAddress.getTownshipCode());
addressList.add(structuredAddress.getCommunity());
addressList.add(structuredAddress.getFormatAddress());
addressList.add(structuredAddress.getIsAgriculture());
addressList.add(structuredAddress.getAddrVerify());
dataMap.put(address, addressList);
}
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
try {
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> addressList = new ArrayList<>();
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
addressList.add(structuredAddress.getProvince());
addressList.add(structuredAddress.getCity());
addressList.add(structuredAddress.getDistrict());
addressList.add(structuredAddress.getTownship());
addressList.add(structuredAddress.getTownshipCode());
addressList.add(structuredAddress.getCommunity());
addressList.add(structuredAddress.getFormatAddress());
addressList.add(structuredAddress.getIsAgriculture());
addressList.add(structuredAddress.getAddrVerify());
dataMap.put(address, addressList);
}
} catch (Exception e) {
System.out.println("give up");
}
} finally {
latch.countDown();
count.incrementAndGet();
synchronized (this) {
if (count.longValue() % 1000 == 0) {
System.out.println("已标准化处理" + count.longValue() + "条地址");
}
}
}
});
}
latch.await();
System.out.println("已完成该批次地址标准化");
for (List<String> data : dataList) {
String address = "";
if (data.size() == column) {
address = data.get(column - 1);
} else {
address = "";
}
List<String> addressRow = dataMap.get(address);
if (CollectionUtils.isNotEmpty(addressRow)) {
data.addAll(addressRow);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
executorService.shutdown();
}
while (true) {
try {
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build();
excelWriter.write(dataList, writeSheet);
excelWriter.finish();
break;
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
Thread.sleep(5000);
}
}
}
private List<List<String>> formatHead(String[] headArr) {
List<List<String>> head = new ArrayList<>();
for (String name: headArr) {
List<String> temp = new ArrayList<>();
temp.add(name);
head.add(temp);
}
return head;
}
}
+53
View File
@@ -0,0 +1,53 @@
public class DivisionStructuredAddress extends StructuredAddress {
private String townshipCode;
private String communityCode;
private String isAgriculture;
private Double bdLng;
private Double bdLat;
public String getCommunityCode() {
return communityCode;
}
public void setCommunityCode(String communityCode) {
this.communityCode = communityCode;
}
public String getIsAgriculture() {
return isAgriculture;
}
public Double getBdLng() {
return bdLng;
}
public void setBdLng(Double bdLng) {
this.bdLng = bdLng;
}
public Double getBdLat() {
return bdLat;
}
public void setBdLat(Double bdLat) {
this.bdLat = bdLat;
}
public void setIsAgriculture(String isAgriculture) {
this.isAgriculture = isAgriculture;
}
public String getTownshipCode() {
return townshipCode;
}
public void setTownshipCode(String townshipCode) {
this.townshipCode = townshipCode;
}
}
+51
View File
@@ -0,0 +1,51 @@
import com.alibaba.excel.EasyExcelFactory;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* excel读写工具类
*
* @author shenjian
* @since 2021/07/12
*/
public class ExcelUtils {
public static List<List<String>> read(InputStream inputStream) {
SimpleExcelListener listener = new SimpleExcelListener();
ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener)
.headRowNumber(1).build();
excelReader.readAll();
List<List<String>> dataList = listener.getDataList();
excelReader.finish();
return dataList;
}
public static class SimpleExcelListener extends AnalysisEventListener {
private List<List<String>> dataList = new ArrayList<>();
@Override
public void invoke(Object o, AnalysisContext analysisContext) {
Map<String, String> map = (Map<String, String>) o;
dataList.add(new ArrayList<>(map.values()));
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
List<List<String>> getDataList() {
return dataList;
}
void setDataList(List<List<String>> dataList) {
this.dataList = dataList;
}
}
}
+124
View File
@@ -0,0 +1,124 @@
import java.io.Serializable;
public class FangJiaApiResponse<T> implements Serializable {
private Integer status;
private Integer msgCode;
private Long count;
private String msg;
private T result;
public FangJiaApiResponse(Integer status, Integer msgCode, String msg) {
this.status = status;
this.msgCode = msgCode;
this.msg = msg;
}
public FangJiaApiResponse() {
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public T getResult() {
return result;
}
public FangJiaApiResponse<T> setResult(T result) {
this.result = result;
return this;
}
public FangJiaApiResponse<T> setCount(Long count) {
this.count = count;
return this;
}
public int getMsgCode() {
return msgCode;
}
public void setMsgCode(int msgCode) {
this.msgCode = msgCode;
}
public Long getCount() {
return count;
}
/**
* 构造一个成功状态的Api Response
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> ok(){
return new FangJiaApiResponse<T>(FangJiaResponseStatus.OK.status, FangJiaResponseMsg.OK_MSG.msgCode, FangJiaResponseMsg.OK_MSG.msg);
}
/**
* 构造一个成功状态的Api Response
* @param result
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> ok(T result){
return new FangJiaApiResponse<T>(FangJiaResponseStatus.OK.status, FangJiaResponseMsg.OK_MSG.msgCode, FangJiaResponseMsg.OK_MSG.msg).setResult(result);
}
/**
* 返回一个参数错误的Api Response
* @param responseMsgCode
* @param msg
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> badRequest(FangJiaResponseMsg responseMsgCode, String msg){
if(msg == null){
msg = responseMsgCode.msg;
}
return new FangJiaApiResponse<T>(FangJiaResponseStatus.FAIL.status, responseMsgCode.msgCode, msg);
}
public static <T> FangJiaApiResponse<T> badRequest(String msg){
return badRequest(FangJiaResponseMsg.BAD_REQUEST_MSG, msg);
}
public static FangJiaApiResponse<Object> serverError(){
return new FangJiaApiResponse<>(FangJiaResponseStatus.FAIL.status,FangJiaResponseMsg.SERVER_ERROR_MSG.msgCode, FangJiaResponseMsg.SERVER_ERROR_MSG.msg);
}
/**
* 返回一个参数错误的Api Response
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> badRequest(){
return badRequest(FangJiaResponseMsg.BAD_REQUEST_MSG, FangJiaResponseMsg.BAD_REQUEST_MSG.msg);
}
/**
* 返回一个License过期的Api Response
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> licenseExpired(){
return new FangJiaApiResponse<T>(FangJiaResponseStatus.FAIL.status, FangJiaResponseMsg.LICENSE_EXPIRED.msgCode, FangJiaResponseMsg.LICENSE_EXPIRED.msg );
}
}
+14
View File
@@ -0,0 +1,14 @@
public enum FangJiaResponseMsg {
OK_MSG(200, "ok"), BAD_REQUEST_MSG(400, "请求参数非法"),
LICENSE_EXPIRED(403, "授权已失效"),
SERVER_ERROR_MSG(500, "服务器异常");
public String msg;
public Integer msgCode;
FangJiaResponseMsg(Integer msgCode, String msg) {
this.msg = msg;
this.msgCode = msgCode;
}
}
+11
View File
@@ -0,0 +1,11 @@
public enum FangJiaResponseStatus {
OK(0, "ok"), FAIL(1, "失败");
public final int status;
public final String msg;
FangJiaResponseStatus(int status, String msg) {
this.status = status;
this.msg = msg;
}
}
+12
View File
@@ -0,0 +1,12 @@
public class RiskLevelInfo {
private String riskLevel;
public String getRiskLevel() {
return riskLevel;
}
public void setRiskLevel(String riskLevel) {
this.riskLevel = riskLevel;
}
}
+146
View File
@@ -0,0 +1,146 @@
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.web.client.RestTemplate;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
/**
* 风险评级测试
*/
public class RiskLevelTest {
static final String IP = "open.fangjia.com";
final String sourceFilePath = "D:\\goojia\\风险评级测试.xlsx";
final String aimFilePath = "D:\\goojia\\风险评级测试结果.xlsx";
final int column = 2;
final Integer nThreads = 1;
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
RiskLevelTest test = new RiskLevelTest();
test.run();
}
public void run() throws FileNotFoundException, InterruptedException {
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
AtomicLong count = new AtomicLong(0);
String[] headArr = new String[]{"ID", "地址", "评级"};
List<List<String>> head = this.formatHead(headArr);
// 默认两行数据
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(sourceFilePath));
if (dataList.size() > 100000) {
throw new RuntimeException("该批次数据量超过100000条");
}
Set<String> set = new HashSet<>();
dataList.forEach(data -> {
if (data.size() == column ) {
set.add(data.get(column - 1));
} else {
// 存在为空情况,需保持一致
set.add("");
data.add("");
}
});
try {
CountDownLatch latch = new CountDownLatch(set.size());
RestTemplate restTemplate = new RestTemplate();
for (String addressTemp : set) {
final String address = addressTemp;
executorService.submit(() -> {
try {
String response = restTemplate.getForObject("http://" + IP + "/address/risk?address={address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if ("ok".equals(apiResponse.getMsg())) {
List<String> addressList = new ArrayList<>();
RiskLevelInfo riskLevelInfo = JSONObject.toJavaObject(apiResponse.getResult(), RiskLevelInfo.class);
addressList.add(riskLevelInfo.getRiskLevel());
dataMap.put(address, addressList);
}
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
String response = restTemplate.getForObject("http://" + IP + "/address/risk?address={address}", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
try {
if ("ok".equals(apiResponse.getMsg())) {
List<String> addressList = new ArrayList<>();
RiskLevelInfo riskLevelInfo = JSONObject.toJavaObject(apiResponse.getResult(), RiskLevelInfo.class);
addressList.add(riskLevelInfo.getRiskLevel());
dataMap.put(address, addressList);
}
} catch (Exception e) {
System.out.println("give up");
}
} finally {
latch.countDown();
count.incrementAndGet();
synchronized (this) {
if (count.longValue() % 1000 == 0) {
System.out.println("已风险评级" + count.longValue() + "条地址");
}
}
}
});
}
latch.await();
System.out.println("已完成该批次地址风险评级");
for (List<String> data : dataList) {
String address = "";
if (data.size() == column) {
address = data.get(column - 1);
} else {
address = "";
}
List<String> addressRow = dataMap.get(address);
if (CollectionUtils.isNotEmpty(addressRow)) {
data.addAll(addressRow);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
executorService.shutdown();
}
while (true) {
try {
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "风险评级结果").head(head).build();
excelWriter.write(dataList, writeSheet);
excelWriter.finish();
break;
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
Thread.sleep(5000);
}
}
}
private List<List<String>> formatHead(String[] headArr) {
List<List<String>> head = new ArrayList<>();
for (String name: headArr) {
List<String> temp = new ArrayList<>();
temp.add(name);
head.add(temp);
}
return head;
}
}
+174
View File
@@ -0,0 +1,174 @@
public class StructuredAddress {
private String province;
private String city;
private String district;
private String dev;
private String township;
private String street;
private String subStreet;
private String streetNumber;
private String community;
private String group;
private String name;
private String buildingNumber;
private String unitNumber;
private String floor;
private String roomNumber;
private Double lng;
private Double lat;
private String formatAddress;
private String addrVerify;
public String getProvince() {
return province;
}
public void setProvince(String province) {
this.province = province;
}
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
public String getTownship() {
return township;
}
public void setTownship(String township) {
this.township = township;
}
public String getStreet() {
return street;
}
public void setStreet(String street) {
this.street = street;
}
public String getStreetNumber() {
return streetNumber;
}
public void setStreetNumber(String streetNumber) {
this.streetNumber = streetNumber;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getBuildingNumber() {
return buildingNumber;
}
public void setBuildingNumber(String buildingNumber) {
this.buildingNumber = buildingNumber;
}
public String getUnitNumber() {
return unitNumber;
}
public void setUnitNumber(String unitNumber) {
this.unitNumber = unitNumber;
}
public String getRoomNumber() {
return roomNumber;
}
public void setRoomNumber(String roomNumber) {
this.roomNumber = roomNumber;
}
public Double getLng() {
return lng;
}
public void setLng(Double lng) {
this.lng = lng;
}
public Double getLat() {
return lat;
}
public void setLat(Double lat) {
this.lat = lat;
}
public String getFloor() {
return floor;
}
public void setFloor(String floor) {
this.floor = floor;
}
public String getCommunity() {
return community;
}
public void setCommunity(String community) {
this.community = community;
}
public String getDev() {
return dev;
}
public void setDev(String dev) {
this.dev = dev;
}
public String getGroup() {
return group;
}
public void setGroup(String group) {
this.group = group;
}
public String getSubStreet() {
return subStreet;
}
public void setSubStreet(String subStreet) {
this.subStreet = subStreet;
}
public String getFormatAddress() {
return formatAddress;
}
public void setFormatAddress(String formatAddress) {
this.formatAddress = formatAddress;
}
public String getAddrVerify() {
return addrVerify;
}
public void setAddrVerify(String addrVerify) {
this.addrVerify = addrVerify;
}
}