Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_gj

This commit is contained in:
2021-09-07 16:44:08 +08:00
10 changed files with 374 additions and 57 deletions
@@ -274,6 +274,7 @@ public class BaiDuApiUtils {
}
return json.getJSONArray("result").getJSONObject(0);
}catch (Exception e){
System.out.println("错误:"+result);
e.printStackTrace();
}
return null;
@@ -34,7 +34,7 @@ class BigdataCommunityBatch extends BaseBatch{
@Override
public void process(Object args) {
def citys = ["宁波"]// CcbCommonUtils.citys;
def citys = CcbCommonUtils.citys;
def rows = []
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
@@ -53,9 +53,9 @@ class BigdataCommunityBatch extends BaseBatch{
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
}
def fileName = "/home/goojia/ccb/bigdata_community.txt"
def fileName = "/home/goojia/ccb/bigdata_community.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat"
}
def allDatas = []
@@ -14,6 +14,7 @@ import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
@@ -27,7 +28,7 @@ import java.util.regex.Pattern
@BatchOwner(Watcher.GaoJuan)
class BigdataDealBatch extends BaseBatch{
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
@@ -69,9 +70,9 @@ class BigdataDealBatch extends BaseBatch{
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
}
def fileName = "/home/goojia/ccb/bigdata_deal.txt"
def fileName = "/home/goojia/ccb/bigdata_deal.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat"
}
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
@@ -399,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{
def startDate = null;
// if(citys.contains(city)){
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
// }else{
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
// }
@@ -407,16 +408,21 @@ class BigdataDealBatch extends BaseBatch{
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
for(def i=0;i<dists.size();i++){
final def dist = dists[i]
// fixedThreadPool.execute(new Runnable() {
// void run() {
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
def price = statMap.get(dist.region+"@"+dist.name)
def startPrice = null
def endPrice = null
if(price){
startPrice = (int)(price - price*0.5)
endPrice = (int)(price*0.5 + price)
if(dist.city=="上海"){
startPrice = (int)(price - price*0.1)
endPrice = (int)(price*0.5 + price)
}else{
startPrice = 1//(int)(price - price*0.5)
endPrice = 500000//(int)(price*0.5 + price)
}
}
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
if(trades){
@@ -428,8 +434,8 @@ class BigdataDealBatch extends BaseBatch{
} finally {
latch.countDown()
}
// };
// })
};
})
}
latch.await()
return tradesList
@@ -447,8 +453,15 @@ class BigdataDealBatch extends BaseBatch{
def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
if(price){
def startPrice = (int)(price - price*0.5)
def endPrice= (int)(price*0.5 + price)
def startPrice = 0
def endPrice= 0
if(obj.fjCity=="上海"){
startPrice = (int)(price - price*0.1)
endPrice = (int)(price*0.5 + price)
}else{
startPrice = 1//(int)(price - price*0.5)
endPrice = 500000//(int)(price*0.5 + price)
}
if(obj.price>=startPrice && obj.price<=endPrice){
rets << obj;
}
@@ -466,7 +479,7 @@ class BigdataDealBatch extends BaseBatch{
q.put("city", city)
q.put("weekend", weekend)
def dbcursor = DIST_SELL_STAT.find(q)
def map = [:]
def map = new ConcurrentHashMap()
for(def d : dbcursor){
map.put(d.region+"@"+d.district_name, d.avg_price)
}
@@ -73,8 +73,8 @@ class BigdataForsaleBatch extends BaseBatch{
index++
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831);
}
}
@@ -144,9 +144,9 @@ class BigdataForsaleBatch extends BaseBatch{
}
latch.await()
def fileName = "/home/goojia/ccb/bigdata_forsale.txt"
def fileName = "/home/goojia/ccb/bigdata_forsale.dat"
if(CommonDatautils.isWindow()){
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt"
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.dat"
}
println city + " 房源总量:"+retMap.size()
@@ -424,7 +424,6 @@ class BigdataForsaleBatch extends BaseBatch{
if (area == null) {
return false;
}
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
return true;
}
@@ -33,37 +33,37 @@ class BigdataPoiBatch extends BaseBatch{
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
def citys = CcbCommonUtils.citys;
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1)
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
def rets = []
def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校")
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows1)
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
rets.addAll(rows1)
// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
// rets.addAll(rows1)
//
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
// rets.addAll(rows2)
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
rets.addAll(rows2)
//
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
// rets.addAll(rows3)
//
// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
// rets.addAll(rows4)
//
// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
// rets.addAll(rows5)
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
rets.addAll(rows4)
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
rets.addAll(rows5)
//
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
// rets.addAll(rows6)
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
if(CommonDatautils.isWindow()){
@@ -119,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{
void run() {
try{
totalCount.addAndGet(1)
if(citys.contains(d['A'])){
if(citys.contains(d['D'])){
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
if(ret){
rets << ret
@@ -219,6 +219,8 @@ class BigdataPoiBatch extends BaseBatch{
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << d['C']
ret << d['F']
ret << "FJW"
return ret;
@@ -226,10 +228,10 @@ class BigdataPoiBatch extends BaseBatch{
def addValue(d, cityCodeDataMap, cityDistMap, type){
def ret = []
def citycode = cityCodeDataMap.get(d['A'])
def citycode = cityCodeDataMap.get(d['D'])
ret << citycode
def distCodeMap = cityDistMap.get(d['A'])
def region =RegionData.formatRegion(d['A'], d['B'])
def distCodeMap = cityDistMap.get(d['D'])
def region =RegionData.formatRegion(d['D'], d['E'])
if(region){
@@ -238,13 +240,13 @@ class BigdataPoiBatch extends BaseBatch{
ret << distObj.district_code
}else{
ret << ""
println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F']
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
}
}else{
ret << ""
}
ret << new ObjectId()
ret << d['C']
ret << d['A']
ret << d['F']
ret << type
if(type=="地铁"){
@@ -258,15 +260,15 @@ class BigdataPoiBatch extends BaseBatch{
}else if(type =="丧葬服务"){
ret << "ob101"
}else if(type =="学校"){
// if(d['F'].contains("中学")){
if(d['F'].contains("中学")){
ret << "ed138"
// }else if(d['F'].contains("小学")){
// ret << "ed114"
// }
}else if(d['F'].contains("小学")){
ret << "ed114"
}
}
if(d['I']){
def lat = d['J'].toDouble()
def lng = d['I'].toDouble()
def lat = d['I'].toDouble()
def lng = d['H'].toDouble()
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
if(loactionMap){
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
@@ -282,6 +284,8 @@ class BigdataPoiBatch extends BaseBatch{
ret << ""
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
ret << ""
ret << ""
ret << "FJW"
return ret;
@@ -174,6 +174,7 @@ class CcbCommonUtils {
q.put("visible",0)
q.put("cat", cat);
q.put("city", city);
// q.put("name", "美辰香醒")
// q.put("name_cat", 0);
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
@@ -186,7 +187,7 @@ class CcbCommonUtils {
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
def rets = []
for(def d : dbsour){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
if(d.address){
d.address = d.address.replaceAll("\r|\n|\\s", "");
}
@@ -0,0 +1,226 @@
package com.fangjia.script.ccbevaluation
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.Sql
import org.apache.lucene.queryparser.classic.QueryParser
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
class DistrictDelete {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
dataSource.setProperties(connectionProperties)
}
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
static main(args) {
def task = new DistrictDelete()
// task.run()
task.gzRun();
}
def run(){
//"泉州","深圳", "重庆",
def citys = ["广州"];//"广州",
def delIndex = 0
for(def city : citys){
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
checkDist(city, dists)
}
}
def checkDist(city,dists){
def rets = Collections.synchronizedList(new ArrayList());
def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd")
def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd")
def totalCount = new AtomicLong()
def delIndex = new AtomicLong()
final CountDownLatch latch = new CountDownLatch(dists.size())
println city +" 小区量:"+dists.size()
def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"]
for(def i=0;i<dists.size();i++) {
final def dist = dists[i];
fixedThreadPool.execute(new Runnable() {
void run() {
try {
totalCount.addAndGet(1)
// def count = findHouses(city, dist.region, "sell", 20190101,20211231, dist.name, dist.alias)
def count = findHouseCount(city, dist.region, dist.name, dist.alias)
def tradeCount = 0
if(!count){
tradeCount = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, 0, 300000)
if(!tradeCount){
def nameFlag = false
for(def key : keys){
if(dist.name.contains(key)){
nameFlag = true
break;
}
}
if(!nameFlag){
if(dist.name.size()<=3 && (dist.name.contains("村") || dist.name.contains("楼") || dist.name.contains("居") )){
nameFlag = true
}
}
def fieldFlag = false
if(dist.finish_building_date && dist.build_type){
fieldFlag = true
}
if(!tradeCount && !count && !fieldFlag){
def rowCount = findCount(dist._id.toString())
if(rowCount || nameFlag){
delIndex.addAndGet(1)
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',q))
println "[ ${delIndex.get()}/${totalCount.get()} ] "+city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
}else{
println city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
}
rets << [dist._id.toString(),dist.city, dist.region, dist.name, dist.address]
}
}
}
} catch (Exception e) {
e.printStackTrace()
} finally {
latch.countDown()
}
}
});
}
latch.await()
ExcelUtils.write("D:\\数据\\建行\\房产估值\\多余小区_${city}.xlsx",rets, "无房源")
}
def findCount(distId){
def sql = "select count(*) as count from fjw_community_artifi_check where commid='${distId}'";
def count = 0
sqlInstance.eachRow(sql,{r->
count = r.count
});
return count
}
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
.must(date_range).must(area_range)
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
if(alias && alias.size()>0){
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
for(def s : alias){
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
}
queryBuilder.must(inBuilder)
}
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
return houses.size();
}
static def findHouseCount(def city, def region, def name, def alias){
if(!alias){
alias = []
}
if(!alias.contains(name)){
alias.add(name)
}
if(name.contains("弄小区")){
def newName = name.replace("弄小区","弄")
if(!alias.contains(newName)){
alias << newName
}
}
def aliasStr = escapeExprSpecialWord(alias.join("','"))
final StringBuffer sqlBuffer = new StringBuffer()
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
sqlBuffer.append(" and city = '"+city+"'")
sqlBuffer.append(" and region = '"+region+"'")
sqlBuffer.append(" and district_name in ('${aliasStr}')")
sqlBuffer.append(" and (cat = 'rent' or cat='sell')")
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
def count = null
while (rs.next()){
count = rs.getInt("count")
}
return count;
}
public static String escapeExprSpecialWord(String keyword) {
return QueryParser.escape(keyword);
}
def gzRun(){
def index = 0
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d->
if(d['U'] && d['U'].toString().toDouble().toInteger()==1){
def id = d['E']
BasicDBObject q=new BasicDBObject('name_cat',1)
q.put("m_date", new Date())
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
index++
println index
}
},0)
}
}
@@ -107,7 +107,7 @@ class NameProcess {
}
}
if(name && !name.contains("路") && !name.contains("街") && !name.contains("大道") && !name.contains("巷")){
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(号|排)"))
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(排)"))
if(h1){
name = name.replaceAll(h1,"")
}
@@ -24,6 +24,7 @@ class SeaweedDataMatchProcess {
rMap.estate_type2 = obj.estate_type2
rMap.city = obj.city
rMap.region = obj.region
rMap.alias = obj.alias
rMap.name = obj.name
rMap.lat = obj.lat/10000000000.0
rMap.lng = obj.lng/10000000000.0
@@ -43,9 +44,6 @@ class SeaweedDataMatchProcess {
}
}
// if(rMap && rMap.id && region.contains(rMap.region)) {
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
// if (obj) {
@@ -0,0 +1,75 @@
package com.fangjia.script.temp.shenjian
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.DateUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
/**
* 快照监控
*
* @author shenjian
* @since 2021/8/27
*/
@BatchOwner(Watcher.ShenJian)
class SnapshotMonitor extends BaseBatch {
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
static void main(String[] args) {
runBatch(new SnapshotMonitor(), args)
}
@Override
void process(Object args) {
// /_snapshot/es_backup/weekly-snap-2021.05*
// 监控两个月的快照
String[] months = [DateUtils.formatDateByPattern(new Date(), "yyyy.MM"), DateUtils.formatDateByPattern(DateUtil.addMonthToDate(new Date(), -1), "yyyy.MM")]
String urlPrefix = "http://192.168.0.206:9200/_snapshot/es_backup/weekly-snap-"
String msg = "";
for (String month : months) {
// 必须分开写,不能有空格
String[] cmds = ["curl", "-XGET", urlPrefix + month + "*", "--user", "elastic:goojia123456"]
String result = execCurl(cmds)
JSONObject jsonObject = JSONObject.parse(result);
if (jsonObject.get("snapshots") != null) {
com.alibaba.fastjson.JSONArray jsonArray = jsonObject.get("snapshots")
for (JSONObject obj : jsonArray) {
String state = obj.get("state")
if (!"SUCCESS".equals(state)) {
msg = "【ES集群快照异常】\n" + obj.get("failures").toString()
DingDingMessageUtil.sendTextMessage(access_token, msg)
return ;
}
}
}
}
}
static String execCurl(String[] cmds) {
ProcessBuilder process = new ProcessBuilder(cmds);
Process p;
try {
p = process.start();
BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
StringBuilder builder = new StringBuilder();
String line = null;
while ((line = reader.readLine()) != null) {
builder.append(line);
builder.append(System.getProperty("line.separator"));
}
return builder.toString();
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
}