新增其他监控
This commit is contained in:
+57
-1
@@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang3.time.DateFormatUtils
|
||||
|
||||
/**
|
||||
* 抓取总量监控
|
||||
@@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch {
|
||||
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SeawaterMonitor(), args)
|
||||
//runBatch(new SeawaterMonitor(), args)
|
||||
//monitorDayCrawlReasonable()
|
||||
def results = []
|
||||
results << ["日期", "网站", "出售", "出租"]
|
||||
for (int i = 1; i < 90; i++) {
|
||||
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime()
|
||||
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
|
||||
countDayCrawlBySite(startDate, endDate, results)
|
||||
}
|
||||
ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1")
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch {
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计近90天更新量超过40万的天数
|
||||
* @param args
|
||||
*/
|
||||
static def monitorDayCrawlReasonable() {
|
||||
def map = [:]
|
||||
for (int i = 1; i < 90; i++) {
|
||||
def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i)
|
||||
def startDate = date.getTime();
|
||||
def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime()
|
||||
def total = countDayCrawl(startDate, endDate)
|
||||
if (total >= expectNumber) {
|
||||
map.put(DateUtil.formatToYYYYMMDD(date), total)
|
||||
}
|
||||
}
|
||||
println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" )
|
||||
for (def key : map.keySet()) {
|
||||
println(key + " : " + map.get(key))
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计每个网站时间段范围内的更新量
|
||||
*/
|
||||
static def countDayCrawlBySite(def startDate, def endDate, def results) {
|
||||
def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"]
|
||||
def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat"
|
||||
def map = [:]
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
map.put(data.root + "|" + data.cat, data.total)
|
||||
});
|
||||
for (def site : sites) {
|
||||
results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")]
|
||||
}
|
||||
return results
|
||||
};
|
||||
|
||||
static def countDayCrawl(def startDate, def endDate) {
|
||||
def total = 0
|
||||
def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'"
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
return total
|
||||
};
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
|
||||
|
||||
+2
-1
@@ -33,7 +33,8 @@ class SeaweedAroundPoi {
|
||||
|
||||
static void main(String[] args) {
|
||||
def types = listAllType()
|
||||
dealSpecialSeaweedAround(types)
|
||||
//dealSpecialSeaweedAround(types)
|
||||
dealPoi(new BasicDBObject(), types)
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user