新增其他监控

This commit is contained in:
沈健
2021-06-11 15:27:00 +08:00
parent 2da2fb9e42
commit e4ac06de8c
2 changed files with 59 additions and 2 deletions
@@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang3.time.DateFormatUtils
/**
* 抓取总量监控
@@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch {
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
static void main(String[] args) {
runBatch(new SeawaterMonitor(), args)
//runBatch(new SeawaterMonitor(), args)
//monitorDayCrawlReasonable()
def results = []
results << ["日期", "网站", "出售", "出租"]
for (int i = 1; i < 90; i++) {
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime()
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
countDayCrawlBySite(startDate, endDate, results)
}
ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1")
}
/**
@@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch {
return ""
}
/**
* 统计近90天更新量超过40万的天数
* @param args
*/
static def monitorDayCrawlReasonable() {
def map = [:]
for (int i = 1; i < 90; i++) {
def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i)
def startDate = date.getTime();
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
def total = countDayCrawl(startDate, endDate)
if (total >= expectNumber) {
map.put(DateUtil.formatToYYYYMMDD(date), total)
}
}
println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" )
for (def key : map.keySet()) {
println(key + " : " + map.get(key))
}
}
/**
* 统计每个网站时间段范围内的更新量
*/
static def countDayCrawlBySite(def startDate, def endDate, def results) {
def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"]
def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat"
def map = [:]
ElasticUtils.executeSqlQuery(sql,{data->
map.put(data.root + "|" + data.cat, data.total)
});
for (def site : sites) {
results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")]
}
return results
};
static def countDayCrawl(def startDate, def endDate) {
def total = 0
def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'"
ElasticUtils.executeSqlQuery(sql,{data->
total = data.total
})
return total
};
@Override
void process(Object args) {
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
@@ -33,7 +33,8 @@ class SeaweedAroundPoi {
static void main(String[] args) {
def types = listAllType()
dealSpecialSeaweedAround(types)
//dealSpecialSeaweedAround(types)
dealPoi(new BasicDBObject(), types)
executorService.shutdown()
}