新增api日志监控
This commit is contained in:
+7
-7
@@ -40,8 +40,8 @@ class TradeStatLog {
|
||||
|
||||
static main(args) {
|
||||
def task = new TradeStatLog()
|
||||
// task.run()//统计最近3个月抓取的成交数据量
|
||||
task.exportTrade()//导出未入库成交
|
||||
task.run()//统计最近3个月抓取的成交数据量
|
||||
// task.exportTrade()//导出未入库成交
|
||||
// task.thirdPartyStat()
|
||||
// println Calendar.getInstance().get(Calendar.YEAR)
|
||||
// task.averageStat()
|
||||
@@ -75,13 +75,13 @@ class TradeStatLog {
|
||||
def run(){
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rets = []
|
||||
rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"]
|
||||
def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"]
|
||||
//def startDate = "2021-05-01"
|
||||
def startDate = "2021-12-01"
|
||||
def date = "2022-01-01"
|
||||
def endDate = "2022-02-01"
|
||||
def newDate = "2022-03-01"
|
||||
def startDate = "2022-01-01"
|
||||
def date = "2022-02-01"
|
||||
def endDate = "2022-03-01"
|
||||
def newDate = "2022-04-01"
|
||||
|
||||
for(def city : citys){
|
||||
def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd"))
|
||||
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.alibaba.excel.util.DateUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.Calendar;
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiTimeOutAnalyBatch extends BaseBatch{
|
||||
|
||||
DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002)
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiTimeOutAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def uriCountMap = findTimeOutUri()
|
||||
sendMessage(uriCountMap)
|
||||
}
|
||||
|
||||
def sendMessage(uriCountMap){
|
||||
def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:"
|
||||
def total = message.length()
|
||||
uriCountMap.each{uri,count->
|
||||
if(count>=5){
|
||||
message = message + uri +"\n"
|
||||
}
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
}
|
||||
|
||||
def findTimeOutUri(){
|
||||
def q = new BasicDBObject()
|
||||
q.put("time", new BasicDBObject('$gte', 2000))
|
||||
def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5)
|
||||
q.put("c_date", new BasicDBObject('$gte', startTime))
|
||||
def dbcursor = LOG.find(q)
|
||||
def uriCountMap = [:]
|
||||
for(def d : dbcursor){
|
||||
def count = uriCountMap.get(d.request_uri)
|
||||
if(!count){
|
||||
count = 0
|
||||
}
|
||||
uriCountMap.put(d.request_uri, count + 1)
|
||||
}
|
||||
return uriCountMap
|
||||
}
|
||||
}
|
||||
+111
@@ -0,0 +1,111 @@
|
||||
package com.fangjia.script.gj.apiLog
|
||||
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import groovy.sql.Sql
|
||||
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ApiUserCountAnalyBatch extends BaseBatch{
|
||||
|
||||
static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver");
|
||||
def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017";
|
||||
DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl)
|
||||
DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl)
|
||||
|
||||
|
||||
static main(args) {
|
||||
runBatch(new ApiUserCountAnalyBatch(), args);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def rets = findAllUsers()
|
||||
sendMessage(rets)
|
||||
}
|
||||
|
||||
|
||||
def sendMessage(def rets){
|
||||
def message = "【Api接口使用提醒】\n"
|
||||
def total = message.length()
|
||||
for(def r : rets){
|
||||
def type = "正式帐户"
|
||||
if(r.type!=0){
|
||||
type = "测试帐户"
|
||||
}
|
||||
message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n"
|
||||
}
|
||||
if(message.length()>total){
|
||||
println message
|
||||
def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea"
|
||||
DingDingMessageUtil.sendTextMessage(access_token, message)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def findAllUsers(){
|
||||
def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'"
|
||||
def rows = dbFangjia.rows(sql)
|
||||
def rets = []
|
||||
for(def row : rows){
|
||||
def obj = findResources(row.id)
|
||||
if(obj){
|
||||
def map = obj.total_count_map
|
||||
def uris = []
|
||||
map.each{k,v->
|
||||
//def visit_count = findVisitCount(row.id, k)
|
||||
def total_visit_count = findAllVisitCount(row.id, k)
|
||||
println row.id +" "+k+" "+v+" "+total_visit_count
|
||||
if(v && total_visit_count){
|
||||
def precent = v*0.8;
|
||||
if(total_visit_count>=precent){
|
||||
uris << k
|
||||
}
|
||||
}
|
||||
}
|
||||
if(uris){
|
||||
rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return rets;
|
||||
}
|
||||
|
||||
def findAllVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date())))
|
||||
def dbcursor = VISIT_COLL.find(q)
|
||||
def total = 0
|
||||
for(def obj : dbcursor){
|
||||
total = total +obj.visit_count
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
|
||||
def findVisitCount(userId, uri){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId.toString())
|
||||
q.put("uri", uri)
|
||||
q.put("visit_date", DateUtils.truncateDay(new Date()))
|
||||
def obj = VISIT_COLL.findOne(q)
|
||||
if(obj){
|
||||
return obj.visit_count
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
def findResources(def userId){
|
||||
def q = new BasicDBObject()
|
||||
q.put("user_id", userId)
|
||||
return AUTHORITY_COLL.findOne(q)
|
||||
}
|
||||
}
|
||||
+2
-2
@@ -182,7 +182,7 @@ public class BeiKeCrawlByChrome {
|
||||
}
|
||||
|
||||
def areaStr = arrs[2].replace("平米", "")
|
||||
if (areaStr) {
|
||||
if (areaStr && areaStr.isNumber()) {
|
||||
detail.area = areaStr.toDouble()
|
||||
}
|
||||
|
||||
@@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles){
|
||||
def href = e.attr("href")
|
||||
if(e.text().contains("福州")){
|
||||
if(!e.text().contains("重庆")){
|
||||
cityMap.put(e.text(), "https:"+href)
|
||||
}
|
||||
}
|
||||
|
||||
+3
-3
@@ -37,7 +37,7 @@ public class LianjiaCrawlByChrome {
|
||||
task.run()
|
||||
// task.run1()
|
||||
task.addDealRegion();
|
||||
// task.standardRegion()
|
||||
task.standardRegion()
|
||||
}
|
||||
|
||||
def run1(){
|
||||
@@ -419,9 +419,9 @@ public class LianjiaCrawlByChrome {
|
||||
if(cityEles){
|
||||
for(def e : cityEles) {
|
||||
def href = e.attr("href")
|
||||
if (!href.contains("fang.lianjia") && e.text().contains("福州")) {
|
||||
// if (!href.contains("fang.lianjia") && e.text().contains("福州")) {
|
||||
cityMap.put(e.text(), href)
|
||||
}
|
||||
// }
|
||||
}
|
||||
}
|
||||
return cityMap
|
||||
|
||||
+1
-1
@@ -18,7 +18,7 @@ class SynchDistrictPrice {
|
||||
// def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房",
|
||||
// "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"]
|
||||
//"fang.com":"搜房",
|
||||
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"]
|
||||
def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","5i5j.com":"我爱我家"]
|
||||
static main(args) {
|
||||
def task = new SynchDistrictPrice()
|
||||
task.run()
|
||||
|
||||
Reference in New Issue
Block a user