diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index 8627efc19..34df24342 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.time.DateFormatUtils /** * 抓取总量监控 @@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch { static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] static void main(String[] args) { - runBatch(new SeawaterMonitor(), args) + //runBatch(new SeawaterMonitor(), args) + //monitorDayCrawlReasonable() + def results = [] + results << ["日期", "网站", "出售", "出租"] + for (int i = 1; i < 90; i++) { + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime() + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + countDayCrawlBySite(startDate, endDate, results) + } + ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1") } /** @@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch { return "" } + /** + * 统计近90天更新量超过40万的天数 + * @param args + */ + static def monitorDayCrawlReasonable() { + def map = [:] + for (int i = 1; i < 90; i++) { + def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i) + def startDate = date.getTime(); + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + def total = countDayCrawl(startDate, endDate) + if (total >= expectNumber) { + map.put(DateUtil.formatToYYYYMMDD(date), total) + } + } + println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" ) + for (def key : map.keySet()) { + println(key + " : " + map.get(key)) + } + } + + /** + * 统计每个网站时间段范围内的更新量 + */ + static def countDayCrawlBySite(def startDate, def endDate, def results) { + def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"] + def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat" + def map = [:] + ElasticUtils.executeSqlQuery(sql,{data-> + map.put(data.root + "|" + data.cat, data.total) + }); + for (def site : sites) { + results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")] + } + return results + }; + + static def countDayCrawl(def startDate, def endDate) { + def total = 0 + def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'" + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + return total + }; + @Override void process(Object args) { def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index addf2d9cf..7a10c9035 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -33,7 +33,8 @@ class SeaweedAroundPoi { static void main(String[] args) { def types = listAllType() - dealSpecialSeaweedAround(types) + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) executorService.shutdown() }