本地化部署测试初始化

This commit is contained in:
沈健
2021-11-09 11:46:43 +08:00
commit d97264166a
32 changed files with 782 additions and 0 deletions
+33
View File
@@ -0,0 +1,33 @@
HELP.md
target/
!.mvn/wrapper/maven-wrapper.jar
!**/src/main/**/target/
!**/src/test/**/target/
### STS ###
.apt_generated
.classpath
.factorypath
.project
.settings
.springBeans
.sts4-cache
### IntelliJ IDEA ###
.idea
*.iws
*.iml
*.ipr
### NetBeans ###
/nbproject/private/
/nbbuild/
/dist/
/nbdist/
/.nb-gradle/
build/
!**/src/main/**/build/
!**/src/test/**/build/
### VS Code ###
.vscode/
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
BIN
View File
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
+164
View File
@@ -0,0 +1,164 @@
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.util.StringUtils;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.web.client.RestClientException;
import org.springframework.web.client.RestTemplate;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
public class BatchTest {
static final String IP = "192.168.0.19:39080";
final String sourceFilePath = "D:\\goojia\\20210715090234.xlsx";
final String aimFilePath = "D:\\goojia\\20210715090234最新结果.xlsx";
final int column = 2;
final Integer nThreads = 20;
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
BatchTest test = new BatchTest();
test.run();
}
public void run() throws FileNotFoundException, InterruptedException {
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
AtomicLong count = new AtomicLong(0);
String[] headArr = new String[]{"ID", "地址", "省份", "城市", "区域", "乡镇街道", "街道编码", "居委村委", "标准化地址", "是否涉农", "地址验证"};
List<List<String>> head = this.formatHead(headArr);
// 默认两行数据
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(sourceFilePath));
if (dataList.size() > 100000) {
throw new RuntimeException("该批次数据量超过100000条");
}
Set<String> set = new HashSet<>();
dataList.forEach(data -> {
if (data.size() == column ) {
set.add(data.get(column - 1));
} else {
// 存在为空情况,需保持一致
set.add("");
data.add("");
}
});
try {
CountDownLatch latch = new CountDownLatch(set.size());
RestTemplate restTemplate = new RestTemplate();
for (String addressTemp : set) {
final String address = addressTemp;
executorService.submit(() -> {
try {
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> addressList = new ArrayList<>();
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
addressList.add(structuredAddress.getProvince());
addressList.add(structuredAddress.getCity());
addressList.add(structuredAddress.getDistrict());
addressList.add(structuredAddress.getTownship());
addressList.add(structuredAddress.getTownshipCode());
addressList.add(structuredAddress.getCommunity());
addressList.add(structuredAddress.getFormatAddress());
addressList.add(structuredAddress.getIsAgriculture());
addressList.add(structuredAddress.getAddrVerify());
dataMap.put(address, addressList);
}
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
String response = restTemplate.getForObject("http://" + IP + "/FangJiaAddressService/fiveDivisionStructure?address={address}", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
try {
if (FangJiaApiResponse.ok().getStatus() == apiResponse.getStatus()) {
List<String> addressList = new ArrayList<>();
DivisionStructuredAddress structuredAddress = JSONObject.toJavaObject(apiResponse.getResult(), DivisionStructuredAddress.class);
addressList.add(structuredAddress.getProvince());
addressList.add(structuredAddress.getCity());
addressList.add(structuredAddress.getDistrict());
addressList.add(structuredAddress.getTownship());
addressList.add(structuredAddress.getTownshipCode());
addressList.add(structuredAddress.getCommunity());
addressList.add(structuredAddress.getFormatAddress());
addressList.add(structuredAddress.getIsAgriculture());
addressList.add(structuredAddress.getAddrVerify());
dataMap.put(address, addressList);
}
} catch (Exception e) {
System.out.println("give up");
}
} finally {
latch.countDown();
count.incrementAndGet();
synchronized (this) {
if (count.longValue() % 1000 == 0) {
System.out.println("已标准化处理" + count.longValue() + "条地址");
}
}
}
});
}
latch.await();
System.out.println("已完成该批次地址标准化");
for (List<String> data : dataList) {
String address = "";
if (data.size() == column) {
address = data.get(column - 1);
} else {
address = "";
}
List<String> addressRow = dataMap.get(address);
if (CollectionUtils.isNotEmpty(addressRow)) {
data.addAll(addressRow);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
executorService.shutdown();
}
while (true) {
try {
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "标准化结果").head(head).build();
excelWriter.write(dataList, writeSheet);
excelWriter.finish();
break;
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
Thread.sleep(5000);
}
}
}
private List<List<String>> formatHead(String[] headArr) {
List<List<String>> head = new ArrayList<>();
for (String name: headArr) {
List<String> temp = new ArrayList<>();
temp.add(name);
head.add(temp);
}
return head;
}
}
+53
View File
@@ -0,0 +1,53 @@
public class DivisionStructuredAddress extends StructuredAddress {
private String townshipCode;
private String communityCode;
private String isAgriculture;
private Double bdLng;
private Double bdLat;
public String getCommunityCode() {
return communityCode;
}
public void setCommunityCode(String communityCode) {
this.communityCode = communityCode;
}
public String getIsAgriculture() {
return isAgriculture;
}
public Double getBdLng() {
return bdLng;
}
public void setBdLng(Double bdLng) {
this.bdLng = bdLng;
}
public Double getBdLat() {
return bdLat;
}
public void setBdLat(Double bdLat) {
this.bdLat = bdLat;
}
public void setIsAgriculture(String isAgriculture) {
this.isAgriculture = isAgriculture;
}
public String getTownshipCode() {
return townshipCode;
}
public void setTownshipCode(String townshipCode) {
this.townshipCode = townshipCode;
}
}
+51
View File
@@ -0,0 +1,51 @@
import com.alibaba.excel.EasyExcelFactory;
import com.alibaba.excel.ExcelReader;
import com.alibaba.excel.context.AnalysisContext;
import com.alibaba.excel.event.AnalysisEventListener;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* excel读写工具类
*
* @author shenjian
* @since 2021/07/12
*/
public class ExcelUtils {
public static List<List<String>> read(InputStream inputStream) {
SimpleExcelListener listener = new SimpleExcelListener();
ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener)
.headRowNumber(1).build();
excelReader.readAll();
List<List<String>> dataList = listener.getDataList();
excelReader.finish();
return dataList;
}
public static class SimpleExcelListener extends AnalysisEventListener {
private List<List<String>> dataList = new ArrayList<>();
@Override
public void invoke(Object o, AnalysisContext analysisContext) {
Map<String, String> map = (Map<String, String>) o;
dataList.add(new ArrayList<>(map.values()));
}
@Override
public void doAfterAllAnalysed(AnalysisContext analysisContext) {
}
List<List<String>> getDataList() {
return dataList;
}
void setDataList(List<List<String>> dataList) {
this.dataList = dataList;
}
}
}
+124
View File
@@ -0,0 +1,124 @@
import java.io.Serializable;
public class FangJiaApiResponse<T> implements Serializable {
private Integer status;
private Integer msgCode;
private Long count;
private String msg;
private T result;
public FangJiaApiResponse(Integer status, Integer msgCode, String msg) {
this.status = status;
this.msgCode = msgCode;
this.msg = msg;
}
public FangJiaApiResponse() {
}
public int getStatus() {
return status;
}
public void setStatus(int status) {
this.status = status;
}
public String getMsg() {
return msg;
}
public void setMsg(String msg) {
this.msg = msg;
}
public T getResult() {
return result;
}
public FangJiaApiResponse<T> setResult(T result) {
this.result = result;
return this;
}
public FangJiaApiResponse<T> setCount(Long count) {
this.count = count;
return this;
}
public int getMsgCode() {
return msgCode;
}
public void setMsgCode(int msgCode) {
this.msgCode = msgCode;
}
public Long getCount() {
return count;
}
/**
* 构造一个成功状态的Api Response
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> ok(){
return new FangJiaApiResponse<T>(FangJiaResponseStatus.OK.status, FangJiaResponseMsg.OK_MSG.msgCode, FangJiaResponseMsg.OK_MSG.msg);
}
/**
* 构造一个成功状态的Api Response
* @param result
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> ok(T result){
return new FangJiaApiResponse<T>(FangJiaResponseStatus.OK.status, FangJiaResponseMsg.OK_MSG.msgCode, FangJiaResponseMsg.OK_MSG.msg).setResult(result);
}
/**
* 返回一个参数错误的Api Response
* @param responseMsgCode
* @param msg
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> badRequest(FangJiaResponseMsg responseMsgCode, String msg){
if(msg == null){
msg = responseMsgCode.msg;
}
return new FangJiaApiResponse<T>(FangJiaResponseStatus.FAIL.status, responseMsgCode.msgCode, msg);
}
public static <T> FangJiaApiResponse<T> badRequest(String msg){
return badRequest(FangJiaResponseMsg.BAD_REQUEST_MSG, msg);
}
public static FangJiaApiResponse<Object> serverError(){
return new FangJiaApiResponse<>(FangJiaResponseStatus.FAIL.status,FangJiaResponseMsg.SERVER_ERROR_MSG.msgCode, FangJiaResponseMsg.SERVER_ERROR_MSG.msg);
}
/**
* 返回一个参数错误的Api Response
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> badRequest(){
return badRequest(FangJiaResponseMsg.BAD_REQUEST_MSG, FangJiaResponseMsg.BAD_REQUEST_MSG.msg);
}
/**
* 返回一个License过期的Api Response
* @param <T>
* @return
*/
public static <T> FangJiaApiResponse<T> licenseExpired(){
return new FangJiaApiResponse<T>(FangJiaResponseStatus.FAIL.status, FangJiaResponseMsg.LICENSE_EXPIRED.msgCode, FangJiaResponseMsg.LICENSE_EXPIRED.msg );
}
}
+14
View File
@@ -0,0 +1,14 @@
public enum FangJiaResponseMsg {
OK_MSG(200, "ok"), BAD_REQUEST_MSG(400, "请求参数非法"),
LICENSE_EXPIRED(403, "授权已失效"),
SERVER_ERROR_MSG(500, "服务器异常");
public String msg;
public Integer msgCode;
FangJiaResponseMsg(Integer msgCode, String msg) {
this.msg = msg;
this.msgCode = msgCode;
}
}
+11
View File
@@ -0,0 +1,11 @@
public enum FangJiaResponseStatus {
OK(0, "ok"), FAIL(1, "失败");
public final int status;
public final String msg;
FangJiaResponseStatus(int status, String msg) {
this.status = status;
this.msg = msg;
}
}
+12
View File
@@ -0,0 +1,12 @@
public class RiskLevelInfo {
private String riskLevel;
public String getRiskLevel() {
return riskLevel;
}
public void setRiskLevel(String riskLevel) {
this.riskLevel = riskLevel;
}
}
+146
View File
@@ -0,0 +1,146 @@
import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.alibaba.fastjson.JSONObject;
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.web.client.RestTemplate;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.atomic.AtomicLong;
/**
* 风险评级测试
*/
public class RiskLevelTest {
static final String IP = "open.fangjia.com";
final String sourceFilePath = "D:\\goojia\\风险评级测试.xlsx";
final String aimFilePath = "D:\\goojia\\风险评级测试结果.xlsx";
final int column = 2;
final Integer nThreads = 1;
public static void main(String[] args) throws FileNotFoundException, InterruptedException {
RiskLevelTest test = new RiskLevelTest();
test.run();
}
public void run() throws FileNotFoundException, InterruptedException {
ExecutorService executorService = Executors.newFixedThreadPool(nThreads);
Map<String, List<String>> dataMap = new ConcurrentHashMap<>();
AtomicLong count = new AtomicLong(0);
String[] headArr = new String[]{"ID", "地址", "评级"};
List<List<String>> head = this.formatHead(headArr);
// 默认两行数据
List<List<String>> dataList = ExcelUtils.read(new FileInputStream(sourceFilePath));
if (dataList.size() > 100000) {
throw new RuntimeException("该批次数据量超过100000条");
}
Set<String> set = new HashSet<>();
dataList.forEach(data -> {
if (data.size() == column ) {
set.add(data.get(column - 1));
} else {
// 存在为空情况,需保持一致
set.add("");
data.add("");
}
});
try {
CountDownLatch latch = new CountDownLatch(set.size());
RestTemplate restTemplate = new RestTemplate();
for (String addressTemp : set) {
final String address = addressTemp;
executorService.submit(() -> {
try {
String response = restTemplate.getForObject("http://" + IP + "/address/risk?address={address}&token=F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
if ("ok".equals(apiResponse.getMsg())) {
List<String> addressList = new ArrayList<>();
RiskLevelInfo riskLevelInfo = JSONObject.toJavaObject(apiResponse.getResult(), RiskLevelInfo.class);
addressList.add(riskLevelInfo.getRiskLevel());
dataMap.put(address, addressList);
}
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
try {
Thread.sleep(5000);
} catch (InterruptedException e) {
}
String response = restTemplate.getForObject("http://" + IP + "/address/risk?address={address}", String.class, address);
FangJiaApiResponse<JSONObject> apiResponse = JSONObject.parseObject(response, FangJiaApiResponse.class);
try {
if ("ok".equals(apiResponse.getMsg())) {
List<String> addressList = new ArrayList<>();
RiskLevelInfo riskLevelInfo = JSONObject.toJavaObject(apiResponse.getResult(), RiskLevelInfo.class);
addressList.add(riskLevelInfo.getRiskLevel());
dataMap.put(address, addressList);
}
} catch (Exception e) {
System.out.println("give up");
}
} finally {
latch.countDown();
count.incrementAndGet();
synchronized (this) {
if (count.longValue() % 1000 == 0) {
System.out.println("已风险评级" + count.longValue() + "条地址");
}
}
}
});
}
latch.await();
System.out.println("已完成该批次地址风险评级");
for (List<String> data : dataList) {
String address = "";
if (data.size() == column) {
address = data.get(column - 1);
} else {
address = "";
}
List<String> addressRow = dataMap.get(address);
if (CollectionUtils.isNotEmpty(addressRow)) {
data.addAll(addressRow);
}
}
} catch (InterruptedException e) {
e.printStackTrace();
} finally {
executorService.shutdown();
}
while (true) {
try {
ExcelWriter excelWriter = EasyExcel.write(aimFilePath).build();
WriteSheet writeSheet = EasyExcel.writerSheet(0, "风险评级结果").head(head).build();
excelWriter.write(dataList, writeSheet);
excelWriter.finish();
break;
} catch (Exception ex) {
ex.printStackTrace();
System.out.println("五秒后重试");
Thread.sleep(5000);
}
}
}
private List<List<String>> formatHead(String[] headArr) {
List<List<String>> head = new ArrayList<>();
for (String name: headArr) {
List<String> temp = new ArrayList<>();
temp.add(name);
head.add(temp);
}
return head;
}
}
+174
View File
@@ -0,0 +1,174 @@
public class StructuredAddress {
private String province;
private String city;
private String district;
private String dev;
private String township;
private String street;
private String subStreet;
private String streetNumber;
private String community;
private String group;
private String name;
private String buildingNumber;
private String unitNumber;
private String floor;
private String roomNumber;
private Double lng;
private Double lat;
private String formatAddress;
private String addrVerify;
public String getProvince() {
return province;
}
public void setProvince(String province) {
this.province = province;
}
public String getCity() {
return city;
}
public void setCity(String city) {
this.city = city;
}
public String getDistrict() {
return district;
}
public void setDistrict(String district) {
this.district = district;
}
public String getTownship() {
return township;
}
public void setTownship(String township) {
this.township = township;
}
public String getStreet() {
return street;
}
public void setStreet(String street) {
this.street = street;
}
public String getStreetNumber() {
return streetNumber;
}
public void setStreetNumber(String streetNumber) {
this.streetNumber = streetNumber;
}
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getBuildingNumber() {
return buildingNumber;
}
public void setBuildingNumber(String buildingNumber) {
this.buildingNumber = buildingNumber;
}
public String getUnitNumber() {
return unitNumber;
}
public void setUnitNumber(String unitNumber) {
this.unitNumber = unitNumber;
}
public String getRoomNumber() {
return roomNumber;
}
public void setRoomNumber(String roomNumber) {
this.roomNumber = roomNumber;
}
public Double getLng() {
return lng;
}
public void setLng(Double lng) {
this.lng = lng;
}
public Double getLat() {
return lat;
}
public void setLat(Double lat) {
this.lat = lat;
}
public String getFloor() {
return floor;
}
public void setFloor(String floor) {
this.floor = floor;
}
public String getCommunity() {
return community;
}
public void setCommunity(String community) {
this.community = community;
}
public String getDev() {
return dev;
}
public void setDev(String dev) {
this.dev = dev;
}
public String getGroup() {
return group;
}
public void setGroup(String group) {
this.group = group;
}
public String getSubStreet() {
return subStreet;
}
public void setSubStreet(String subStreet) {
this.subStreet = subStreet;
}
public String getFormatAddress() {
return formatAddress;
}
public void setFormatAddress(String formatAddress) {
this.formatAddress = formatAddress;
}
public String getAddrVerify() {
return addrVerify;
}
public void setAddrVerify(String addrVerify) {
this.addrVerify = addrVerify;
}
}