新增抓取运维警报
This commit is contained in:
@@ -32,6 +32,7 @@ public enum Watcher {
|
||||
BaYi('15300424133','bayi@fangjia.com'),
|
||||
WeiLei('18101751850','weilei@fangjia.com'),
|
||||
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
|
||||
ShenJian('13764369683', 'shenjian@fangjia.com')
|
||||
private final String mobile;
|
||||
private final String email;
|
||||
|
||||
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.time.DateUtils
|
||||
|
||||
/**
|
||||
* 抓取总量监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/12
|
||||
*/
|
||||
//@BatchOwner(Watcher.ShenJian)
|
||||
//class SeawaterMonitor extends BaseBatch {
|
||||
class SeawaterMonitor {
|
||||
|
||||
static final Long expectNumber = 400000;
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
// 重点城市
|
||||
static final city = ["北京','唐山','天津','秦皇岛','石家庄','包头','太原','丹东','呼和浩特','锦州'," +
|
||||
"'沈阳','吉林','大连','牡丹江','长春','无锡','哈尔滨','扬州','上海','徐州'," +
|
||||
"'南京','温州','杭州','金华','宁波','蚌埠','合肥','安庆','福州','泉州'," +
|
||||
"'厦门','九江','南昌','赣州','济南','烟台','青岛','济宁','郑州','洛阳'," +
|
||||
"'武汉','平顶山','长沙','宜昌','广州','襄阳','深圳','岳阳','南宁','常德'," +
|
||||
"'海口','惠州','重庆','湛江','成都','韶关','贵阳','桂林','昆明','北海'," +
|
||||
"'西安','三亚','兰州','泸州','西宁','南充','银川','遵义','乌鲁木齐','大理'," +
|
||||
"'东莞','佛山','苏州','珠海','中山','台州','绍兴','南通','常州','嘉兴'," +
|
||||
"'淄博','株洲','镇江','肇庆','漳州','盐城','芜湖','泰州','汕头','清远'," +
|
||||
"'马鞍山','柳州','连云港','廊坊','淮安','湖州','沧州','保定','张家口','日照'," +
|
||||
"'拉萨','大同','文昌','陵水','保亭"];
|
||||
// 重点网站
|
||||
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
|
||||
|
||||
static void main(String[] args) {
|
||||
def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd")
|
||||
def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd")
|
||||
def startDate = yesterday + "T00:00:00"
|
||||
def endDate = yesterday + "T23:59:59"
|
||||
|
||||
def result = []
|
||||
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorCityCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
if (result.size() > 0) {
|
||||
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
|
||||
println(msg)
|
||||
DingDingMessageUtil.sendTextMessage(access_token, msg)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 监控前一日抓取量
|
||||
* @return
|
||||
*/
|
||||
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
|
||||
def total
|
||||
String sql = "select count(*) as total from latest_seawater where m_date between '${startDate}' and '${endDate}' "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
if (total < expectNumber) {
|
||||
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点城市近半月抓取量
|
||||
* @return
|
||||
*/
|
||||
static boolean countCityCrawlNumber(def startDate, def endDate) {
|
||||
def result = []
|
||||
result << ['城市', '抓取量']
|
||||
// 统计近半个月各重点城市抓取量
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
result << [data.city, data.total]
|
||||
})
|
||||
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计前一日重点城市抓取量,昨天为零则警报
|
||||
* @return
|
||||
*/
|
||||
static def monitorCityCrawlNumber(def startDate, def endDate) {
|
||||
def msg = "【重点城市抓取异常】"
|
||||
def unNormal = []
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) where total=0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
unNormal.add(data.city)
|
||||
})
|
||||
msg += unNormal.join("、") + " 抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点网站抓取情况,近一周为0则警报
|
||||
*/
|
||||
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
|
||||
def msg = "【重点网站抓取异常】"
|
||||
def set = new HashSet()
|
||||
def siteFormatted = site.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, root from latest_seawater where m_date between '${lastWeekDate}' and '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.root)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : site) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 近一周抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// @Override
|
||||
// void process(Object args) {
|
||||
//
|
||||
// }
|
||||
}
|
||||
Reference in New Issue
Block a user