diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index afbaed5ee..18a865ad1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 - task.exportTrade()//导出未入库成交 + task.run()//统计最近3个月抓取的成交数据量 +// task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","1月总量","2月总量","3月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2021-12-01" - def date = "2022-01-01" - def endDate = "2022-02-01" - def newDate = "2022-03-01" + def startDate = "2022-01-01" + def date = "2022-02-01" + def endDate = "2022-03-01" + def newDate = "2022-04-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy new file mode 100644 index 000000000..b002b2c3b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiTimeOutAnalyBatch.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.gj.apiLog + +import com.alibaba.excel.util.DateUtils +import com.fangjia.core.util.DateUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.Calendar; + +@BatchOwner(Watcher.GaoJuan) +class ApiTimeOutAnalyBatch extends BaseBatch{ + + DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) + + static main(args) { + runBatch(new ApiTimeOutAnalyBatch(), args); + } + + @Override + public void process(Object args) { + def uriCountMap = findTimeOutUri() + sendMessage(uriCountMap) + } + + def sendMessage(uriCountMap){ + def message = "【Api接口近5分钟超过5条请求时间超3秒提醒】\n超时接口:" + def total = message.length() + uriCountMap.each{uri,count-> + if(count>=5){ + message = message + uri +"\n" + } + } + if(message.length()>total){ + println message + def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea" + DingDingMessageUtil.sendTextMessage(access_token, message) + } + } + + def findTimeOutUri(){ + def q = new BasicDBObject() + q.put("time", new BasicDBObject('$gte', 2000)) + def startTime = DateUtil.add(new Date(), Calendar.MINUTE, -5) + q.put("c_date", new BasicDBObject('$gte', startTime)) + def dbcursor = LOG.find(q) + def uriCountMap = [:] + for(def d : dbcursor){ + def count = uriCountMap.get(d.request_uri) + if(!count){ + count = 0 + } + uriCountMap.put(d.request_uri, count + 1) + } + return uriCountMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy new file mode 100644 index 000000000..16063d88b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/apiLog/ApiUserCountAnalyBatch.groovy @@ -0,0 +1,111 @@ +package com.fangjia.script.gj.apiLog + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.sql.Sql + +@BatchOwner(Watcher.GaoJuan) +class ApiUserCountAnalyBatch extends BaseBatch{ + + static final dbFangjia = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia?autoReconnect=true&characterEncoding=utf-8&autoReconnect=true", "sa", "G00jia", "com.mysql.jdbc.Driver"); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection AUTHORITY_COLL = MongoUtils.getCollectionByName2("fangjia", "authority.api_user_authority", lineUrl) + DBCollection VISIT_COLL = MongoUtils.getCollectionByName2("business", "api_visit_count", lineUrl) + + + static main(args) { + runBatch(new ApiUserCountAnalyBatch(), args); + } + + @Override + public void process(Object args) { + def rets = findAllUsers() + sendMessage(rets) + } + + + def sendMessage(def rets){ + def message = "【Api接口使用提醒】\n" + def total = message.length() + for(def r : rets){ + def type = "正式帐户" + if(r.type!=0){ + type = "测试帐户" + } + message = message + r.company+ " 用户ID:"+r.user_id+"(${type}),有 "+r.uris.size()+" 个接口使用率已超过80%,接口有:"+r.uris.join(",")+"\n" + } + if(message.length()>total){ + println message + def access_token = "1e25f620ea2be926b2441e4f30d67f37abdff2e14088b03b41b4082b11fc34ea" + DingDingMessageUtil.sendTextMessage(access_token, message) + } + + } + + def findAllUsers(){ + def sql = "select id,COMPANY,is_test from api_user where dead_time>='"+DateUtils.formatDateByPattern(new Date(),'yyyy-MM-dd')+"'" + def rows = dbFangjia.rows(sql) + def rets = [] + for(def row : rows){ + def obj = findResources(row.id) + if(obj){ + def map = obj.total_count_map + def uris = [] + map.each{k,v-> + //def visit_count = findVisitCount(row.id, k) + def total_visit_count = findAllVisitCount(row.id, k) + println row.id +" "+k+" "+v+" "+total_visit_count + if(v && total_visit_count){ + def precent = v*0.8; + if(total_visit_count>=precent){ + uris << k + } + } + } + if(uris){ + rets << ["user_id":row.id,"company":row.company,"uris":uris,"type":row.is_test] + } + } + } + + return rets; + } + + def findAllVisitCount(userId, uri){ + def q = new BasicDBObject() + q.put("user_id", userId.toString()) + q.put("uri", uri) + q.put("visit_date", new BasicDBObject('$lte',DateUtils.truncateDay(new Date()))) + def dbcursor = VISIT_COLL.find(q) + def total = 0 + for(def obj : dbcursor){ + total = total +obj.visit_count + } + return total + } + + + def findVisitCount(userId, uri){ + def q = new BasicDBObject() + q.put("user_id", userId.toString()) + q.put("uri", uri) + q.put("visit_date", DateUtils.truncateDay(new Date())) + def obj = VISIT_COLL.findOne(q) + if(obj){ + return obj.visit_count + } + return null + } + + def findResources(def userId){ + def q = new BasicDBObject() + q.put("user_id", userId) + return AUTHORITY_COLL.findOne(q) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index d54ad9fb3..62fe7b599 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -182,7 +182,7 @@ public class BeiKeCrawlByChrome { } def areaStr = arrs[2].replace("平米", "") - if (areaStr) { + if (areaStr && areaStr.isNumber()) { detail.area = areaStr.toDouble() } @@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("福州")){ + if(!e.text().contains("重庆")){ cityMap.put(e.text(), "https:"+href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index f1750cfa9..5c8d9c178 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -37,7 +37,7 @@ public class LianjiaCrawlByChrome { task.run() // task.run1() task.addDealRegion(); -// task.standardRegion() + task.standardRegion() } def run1(){ @@ -419,9 +419,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles) { def href = e.attr("href") - if (!href.contains("fang.lianjia") && e.text().contains("福州")) { +// if (!href.contains("fang.lianjia") && e.text().contains("福州")) { cityMap.put(e.text(), href) - } +// } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 795b1556a..e308339de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,8 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ; + // 要导出的月份第一天 + Date now = DateUtils.parseFromTightByPattern("2022-03-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy new file mode 100644 index 000000000..c23db19ed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromRegionBlock.groovy @@ -0,0 +1,27 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +final DBCollection ZS_REGION_CONN = MongoUtils.getCollectionByName2("fangjia", "region_block", ZS_MONGODB_URL) + +ZS_REGION_CONN.find(new BasicDBObject()).batchSize(1000).each {obj -> + def city = obj.city + def org_name = obj.org_name + + List result = GaoDeUtils.searchPoi(city, org_name, org_name, null) + if (CollectionUtils.isNotEmpty(result)) { + def location = result.get(0).location + def lon = location.split(",")[0] + def lat = location.split(",")[1] + ZS_REGION_CONN.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set',new BasicDBObject("location", new BasicDBObject("lon", lon).append("lat", lat)))) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 279f5f3c1..12d8ebc60 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家","jiwu.com":"吉屋"] //"fang.com":"搜房", - def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","cityhouse.cn":"城市房产"] + def sourceMap = ["ke.com":"贝壳找房","fang.com":"搜房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run()