From 1f0a3177199d152e19f24d546a4529d2f1b78b0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 16:46:56 +0800 Subject: [PATCH 1/3] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E8=AD=A6=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/Watcher.groovy | 1 + .../temp/shenjian/SeawaterMonitor.groovy | 148 ++++++++++++++++++ 2 files changed, 149 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy index 85088ddd8..004d6bd4a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/Watcher.groovy @@ -32,6 +32,7 @@ public enum Watcher { BaYi('15300424133','bayi@fangjia.com'), WeiLei('18101751850','weilei@fangjia.com'), JiangYaHui('13684913780','jiangyahui@fangjia.com'), + ShenJian('13764369683', 'shenjian@fangjia.com') private final String mobile; private final String email; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy new file mode 100644 index 000000000..a2d5c158d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -0,0 +1,148 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import org.apache.commons.lang.StringUtils +import org.apache.commons.lang.time.DateUtils + +/** + * 抓取总量监控 + * + * @author shenjian + * @since 2021/5/12 + */ +//@BatchOwner(Watcher.ShenJian) +//class SeawaterMonitor extends BaseBatch { +class SeawaterMonitor { + + static final Long expectNumber = 400000; + // 运维组 运维警报 + static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + // 重点城市 + static final city = ["北京','唐山','天津','秦皇岛','石家庄','包头','太原','丹东','呼和浩特','锦州'," + + "'沈阳','吉林','大连','牡丹江','长春','无锡','哈尔滨','扬州','上海','徐州'," + + "'南京','温州','杭州','金华','宁波','蚌埠','合肥','安庆','福州','泉州'," + + "'厦门','九江','南昌','赣州','济南','烟台','青岛','济宁','郑州','洛阳'," + + "'武汉','平顶山','长沙','宜昌','广州','襄阳','深圳','岳阳','南宁','常德'," + + "'海口','惠州','重庆','湛江','成都','韶关','贵阳','桂林','昆明','北海'," + + "'西安','三亚','兰州','泸州','西宁','南充','银川','遵义','乌鲁木齐','大理'," + + "'东莞','佛山','苏州','珠海','中山','台州','绍兴','南通','常州','嘉兴'," + + "'淄博','株洲','镇江','肇庆','漳州','盐城','芜湖','泰州','汕头','清远'," + + "'马鞍山','柳州','连云港','廊坊','淮安','湖州','沧州','保定','张家口','日照'," + + "'拉萨','大同','文昌','陵水','保亭"]; + // 重点网站 + static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] + + static void main(String[] args) { + def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") + def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") + def startDate = yesterday + "T00:00:00" + def endDate = yesterday + "T23:59:59" + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } + + /** + * 监控前一日抓取量 + * @return + */ + static def monitorYesterdayCrawlNumber(def startDate, def endDate) { + def total + String sql = "select count(*) as total from latest_seawater where m_date between '${startDate}' and '${endDate}' " + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + if (total < expectNumber) { + return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber + } + return "" + } + + /** + * 统计重点城市近半月抓取量 + * @return + */ + static boolean countCityCrawlNumber(def startDate, def endDate) { + def result = [] + result << ['城市', '抓取量'] + // 统计近半个月各重点城市抓取量 + def cityFormatted = city.join("','") + String sql = "select * from (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + ElasticUtils.executeSqlQuery(sql,{data-> + result << [data.city, data.total] + }) + ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量") + } + + /** + * 统计前一日重点城市抓取量,昨天为零则警报 + * @return + */ + static def monitorCityCrawlNumber(def startDate, def endDate) { + def msg = "【重点城市抓取异常】" + def unNormal = [] + def cityFormatted = city.join("','") + String sql = "select * from " + + " (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) where total=0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + unNormal.add(data.city) + }) + msg += unNormal.join("、") + " 抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + + /** + * 统计重点网站抓取情况,近一周为0则警报 + */ + static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) { + def msg = "【重点网站抓取异常】" + def set = new HashSet() + def siteFormatted = site.join("','") + String sql = "select * from " + + " (select count(*) as total, root from latest_seawater where m_date between '${lastWeekDate}' and '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + //" order by total " + ElasticUtils.executeSqlQuery(sql,{data-> + set.add(data.root) + }) + def unNormal = [] + for (def info : site) { + if (!set.contains(info)) { + unNormal.add(info) + } + } + msg += unNormal.join("、") + " 近一周抓取量为0" + if (unNormal.size() > 0) { + return msg + } + return "" + } + +// @Override +// void process(Object args) { +// +// } +} From c0484532090cdce60d8bc20a7f93ecfaf35e9ab8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 18:43:20 +0800 Subject: [PATCH 2/3] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E8=BF=90=E7=BB=B4=E8=AD=A6=E6=8A=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 66 ++++++++++--------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index a2d5c158d..60e2419de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -1,5 +1,8 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil @@ -12,9 +15,8 @@ import org.apache.commons.lang.time.DateUtils * @author shenjian * @since 2021/5/12 */ -//@BatchOwner(Watcher.ShenJian) -//class SeawaterMonitor extends BaseBatch { -class SeawaterMonitor { +@BatchOwner(Watcher.ShenJian) +class SeawaterMonitor extends BaseBatch { static final Long expectNumber = 400000; // 运维组 运维警报 @@ -35,31 +37,7 @@ class SeawaterMonitor { static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] static void main(String[] args) { - def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") - def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") - def startDate = yesterday + "T00:00:00" - def endDate = yesterday + "T23:59:59" - - def result = [] - def msg = monitorYesterdayCrawlNumber(startDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - - msg = monitorCityCrawlNumber(startDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - - msg = monitorSiteCrawlNumber(lastWeekDate, endDate) - if (StringUtils.isNotBlank(msg)) { - result.add(msg) - } - if (result.size() > 0) { - msg = "【日期】 " + yesterday + "\n" + result.join("\n") - println(msg) - DingDingMessageUtil.sendTextMessage(access_token, msg) - } + runBatch(new SeawaterMonitor(), args) } /** @@ -141,8 +119,32 @@ class SeawaterMonitor { return "" } -// @Override -// void process(Object args) { -// -// } + @Override + void process(Object args) { + def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") + def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") + def startDate = yesterday + "T00:00:00" + def endDate = yesterday + "T23:59:59" + + def result = [] + def msg = monitorYesterdayCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorCityCrawlNumber(startDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + + msg = monitorSiteCrawlNumber(lastWeekDate, endDate) + if (StringUtils.isNotBlank(msg)) { + result.add(msg) + } + if (result.size() > 0) { + msg = "【日期】 " + yesterday + "\n" + result.join("\n") + println(msg) + DingDingMessageUtil.sendTextMessage(access_token, msg) + } + } } From b8d282112291ed0de9bbb0cb1325b5c53753c5ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 09:46:19 +0800 Subject: [PATCH 3/3] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A5=E6=9C=9F?= =?UTF-8?q?=E5=8F=8A=E9=87=8D=E7=82=B9=E5=9F=8E=E5=B8=82=E6=9F=A5=E8=AF=A2?= =?UTF-8?q?bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index 60e2419de..8627efc19 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher @@ -7,7 +8,6 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import org.apache.commons.lang.StringUtils -import org.apache.commons.lang.time.DateUtils /** * 抓取总量监控 @@ -22,17 +22,17 @@ class SeawaterMonitor extends BaseBatch { // 运维组 运维警报 static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" // 重点城市 - static final city = ["北京','唐山','天津','秦皇岛','石家庄','包头','太原','丹东','呼和浩特','锦州'," + - "'沈阳','吉林','大连','牡丹江','长春','无锡','哈尔滨','扬州','上海','徐州'," + - "'南京','温州','杭州','金华','宁波','蚌埠','合肥','安庆','福州','泉州'," + - "'厦门','九江','南昌','赣州','济南','烟台','青岛','济宁','郑州','洛阳'," + - "'武汉','平顶山','长沙','宜昌','广州','襄阳','深圳','岳阳','南宁','常德'," + - "'海口','惠州','重庆','湛江','成都','韶关','贵阳','桂林','昆明','北海'," + - "'西安','三亚','兰州','泸州','西宁','南充','银川','遵义','乌鲁木齐','大理'," + - "'东莞','佛山','苏州','珠海','中山','台州','绍兴','南通','常州','嘉兴'," + - "'淄博','株洲','镇江','肇庆','漳州','盐城','芜湖','泰州','汕头','清远'," + - "'马鞍山','柳州','连云港','廊坊','淮安','湖州','沧州','保定','张家口','日照'," + - "'拉萨','大同','文昌','陵水','保亭"]; + static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州", + "沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州", + "南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州", + "厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳", + "武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德", + "海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海", + "西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理", + "东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴", + "淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远", + "马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照", + "拉萨","大同","文昌","陵水","保亭"]; // 重点网站 static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] @@ -46,7 +46,7 @@ class SeawaterMonitor extends BaseBatch { */ static def monitorYesterdayCrawlNumber(def startDate, def endDate) { def total - String sql = "select count(*) as total from latest_seawater where m_date between '${startDate}' and '${endDate}' " + String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' " ElasticUtils.executeSqlQuery(sql,{data-> total = data.total }) @@ -65,7 +65,7 @@ class SeawaterMonitor extends BaseBatch { result << ['城市', '抓取量'] // 统计近半个月各重点城市抓取量 def cityFormatted = city.join("','") - String sql = "select * from (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " + String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc " ElasticUtils.executeSqlQuery(sql,{data-> result << [data.city, data.total] }) @@ -78,14 +78,20 @@ class SeawaterMonitor extends BaseBatch { */ static def monitorCityCrawlNumber(def startDate, def endDate) { def msg = "【重点城市抓取异常】" - def unNormal = [] + def set = new HashSet() def cityFormatted = city.join("','") String sql = "select * from " + - " (select count(*) as total, city from latest_seawater where m_date between '${startDate}' and '${endDate}' and city in('${cityFormatted}') group by city) where total=0 " + " (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 " //" order by total " ElasticUtils.executeSqlQuery(sql,{data-> - unNormal.add(data.city) + set.add(data.city) }) + def unNormal = [] + for (def info : city) { + if (!set.contains(info)) { + unNormal.add(info) + } + } msg += unNormal.join("、") + " 抓取量为0" if (unNormal.size() > 0) { return msg @@ -101,7 +107,7 @@ class SeawaterMonitor extends BaseBatch { def set = new HashSet() def siteFormatted = site.join("','") String sql = "select * from " + - " (select count(*) as total, root from latest_seawater where m_date between '${lastWeekDate}' and '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " + " (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 " //" order by total " ElasticUtils.executeSqlQuery(sql,{data-> set.add(data.root) @@ -121,10 +127,10 @@ class SeawaterMonitor extends BaseBatch { @Override void process(Object args) { - def lastWeekDate = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -7), "yyyy-MM-dd") - def yesterday = org.apache.commons.lang3.time.DateFormatUtils.format(DateUtils.addDays(new Date(), -1), "yyyy-MM-dd") - def startDate = yesterday + "T00:00:00" - def endDate = yesterday + "T23:59:59" + def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() + def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1)) + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime(); + def endDate = DateUtil.truncateDay(new Date()).getTime() def result = [] def msg = monitorYesterdayCrawlNumber(startDate, endDate)