This commit is contained in:
2021-05-18 12:52:01 +08:00
2 changed files with 157 additions and 0 deletions
@@ -32,6 +32,7 @@ public enum Watcher {
BaYi('15300424133','bayi@fangjia.com'),
WeiLei('18101751850','weilei@fangjia.com'),
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
ShenJian('13764369683', 'shenjian@fangjia.com')
private final String mobile;
private final String email;
@@ -0,0 +1,156 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import org.apache.commons.lang.StringUtils
/**
* 抓取总量监控
*
* @author shenjian
* @since 2021/5/12
*/
@BatchOwner(Watcher.ShenJian)
class SeawaterMonitor extends BaseBatch {
static final Long expectNumber = 400000;
// 运维组 运维警报
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
// 重点城市
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
"拉萨","大同","文昌","陵水","保亭"];
// 重点网站
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
static void main(String[] args) {
runBatch(new SeawaterMonitor(), args)
}
/**
* 监控前一日抓取量
* @return
*/
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
def total
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
if (total < expectNumber) {
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
}
return ""
}
/**
* 统计重点城市近半月抓取量
* @return
*/
static boolean countCityCrawlNumber(def startDate, def endDate) {
def result = []
result << ['城市', '抓取量']
// 统计近半个月各重点城市抓取量
def cityFormatted = city.join("','")
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
ElasticUtils.executeSqlQuery(sql,{data->
result << [data.city, data.total]
})
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
}
/**
* 统计前一日重点城市抓取量,昨天为零则警报
* @return
*/
static def monitorCityCrawlNumber(def startDate, def endDate) {
def msg = "【重点城市抓取异常】"
def set = new HashSet()
def cityFormatted = city.join("','")
String sql = "select * from " +
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.city)
})
def unNormal = []
for (def info : city) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
/**
* 统计重点网站抓取情况,近一周为0则警报
*/
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
def msg = "【重点网站抓取异常】"
def set = new HashSet()
def siteFormatted = site.join("','")
String sql = "select * from " +
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
//" order by total "
ElasticUtils.executeSqlQuery(sql,{data->
set.add(data.root)
})
def unNormal = []
for (def info : site) {
if (!set.contains(info)) {
unNormal.add(info)
}
}
msg += unNormal.join("、") + " 近一周抓取量为0"
if (unNormal.size() > 0) {
return msg
}
return ""
}
@Override
void process(Object args) {
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
def endDate = DateUtil.truncateDay(new Date()).getTime()
def result = []
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorCityCrawlNumber(startDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
if (StringUtils.isNotBlank(msg)) {
result.add(msg)
}
if (result.size() > 0) {
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
println(msg)
DingDingMessageUtil.sendTextMessage(access_token, msg)
}
}
}