Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into script_gj
This commit is contained in:
@@ -32,6 +32,7 @@ public enum Watcher {
|
||||
BaYi('15300424133','bayi@fangjia.com'),
|
||||
WeiLei('18101751850','weilei@fangjia.com'),
|
||||
JiangYaHui('13684913780','jiangyahui@fangjia.com'),
|
||||
ShenJian('13764369683', 'shenjian@fangjia.com')
|
||||
private final String mobile;
|
||||
private final String email;
|
||||
|
||||
|
||||
+156
@@ -0,0 +1,156 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
|
||||
/**
|
||||
* 抓取总量监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/5/12
|
||||
*/
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SeawaterMonitor extends BaseBatch {
|
||||
|
||||
static final Long expectNumber = 400000;
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
// 重点城市
|
||||
static final city = ["北京","唐山","天津","秦皇岛","石家庄","包头","太原","丹东","呼和浩特","锦州",
|
||||
"沈阳","吉林","大连","牡丹江","长春","无锡","哈尔滨","扬州","上海","徐州",
|
||||
"南京","温州","杭州","金华","宁波","蚌埠","合肥","安庆","福州","泉州",
|
||||
"厦门","九江","南昌","赣州","济南","烟台","青岛","济宁","郑州","洛阳",
|
||||
"武汉","平顶山","长沙","宜昌","广州","襄阳","深圳","岳阳","南宁","常德",
|
||||
"海口","惠州","重庆","湛江","成都","韶关","贵阳","桂林","昆明","北海",
|
||||
"西安","三亚","兰州","泸州","西宁","南充","银川","遵义","乌鲁木齐","大理",
|
||||
"东莞","佛山","苏州","珠海","中山","台州","绍兴","南通","常州","嘉兴",
|
||||
"淄博","株洲","镇江","肇庆","漳州","盐城","芜湖","泰州","汕头","清远",
|
||||
"马鞍山","柳州","连云港","廊坊","淮安","湖州","沧州","保定","张家口","日照",
|
||||
"拉萨","大同","文昌","陵水","保亭"];
|
||||
// 重点网站
|
||||
static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"]
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SeawaterMonitor(), args)
|
||||
}
|
||||
|
||||
/**
|
||||
* 监控前一日抓取量
|
||||
* @return
|
||||
*/
|
||||
static def monitorYesterdayCrawlNumber(def startDate, def endDate) {
|
||||
def total
|
||||
String sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}' "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
total = data.total
|
||||
})
|
||||
if (total < expectNumber) {
|
||||
return "【总量抓取异常】 抓取量为" + total + ", 低于预期值" + expectNumber
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点城市近半月抓取量
|
||||
* @return
|
||||
*/
|
||||
static boolean countCityCrawlNumber(def startDate, def endDate) {
|
||||
def result = []
|
||||
result << ['城市', '抓取量']
|
||||
// 统计近半个月各重点城市抓取量
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) order by total desc "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
result << [data.city, data.total]
|
||||
})
|
||||
ExcelUtils.write("F:\\数据\\重点城市抓取\\近半月重点城市抓取量.xlsx", result, "抓取量")
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计前一日重点城市抓取量,昨天为零则警报
|
||||
* @return
|
||||
*/
|
||||
static def monitorCityCrawlNumber(def startDate, def endDate) {
|
||||
def msg = "【重点城市抓取异常】"
|
||||
def set = new HashSet()
|
||||
def cityFormatted = city.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, city from seawater where m_date >= '${startDate}' and m_date < '${endDate}' and city in('${cityFormatted}') group by city) where total>0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.city)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : city) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
/**
|
||||
* 统计重点网站抓取情况,近一周为0则警报
|
||||
*/
|
||||
static def monitorSiteCrawlNumber(def lastWeekDate, def endDate) {
|
||||
def msg = "【重点网站抓取异常】"
|
||||
def set = new HashSet()
|
||||
def siteFormatted = site.join("','")
|
||||
String sql = "select * from " +
|
||||
" (select count(*) as total, root from seawater where m_date >= '${lastWeekDate}' and m_date < '${endDate}' and root in('${siteFormatted}') group by root) where total > 0 "
|
||||
//" order by total "
|
||||
ElasticUtils.executeSqlQuery(sql,{data->
|
||||
set.add(data.root)
|
||||
})
|
||||
def unNormal = []
|
||||
for (def info : site) {
|
||||
if (!set.contains(info)) {
|
||||
unNormal.add(info)
|
||||
}
|
||||
}
|
||||
msg += unNormal.join("、") + " 近一周抓取量为0"
|
||||
if (unNormal.size() > 0) {
|
||||
return msg
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime()
|
||||
def yesterday = DateUtil.formatToYYYYMMDD(DateUtil.add(DateUtil.truncateDay(new Date()), -1))
|
||||
def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1).getTime();
|
||||
def endDate = DateUtil.truncateDay(new Date()).getTime()
|
||||
|
||||
def result = []
|
||||
def msg = monitorYesterdayCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorCityCrawlNumber(startDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
|
||||
msg = monitorSiteCrawlNumber(lastWeekDate, endDate)
|
||||
if (StringUtils.isNotBlank(msg)) {
|
||||
result.add(msg)
|
||||
}
|
||||
if (result.size() > 0) {
|
||||
msg = "【日期】 " + yesterday + "\n" + result.join("\n")
|
||||
println(msg)
|
||||
DingDingMessageUtil.sendTextMessage(access_token, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -4,36 +4,79 @@ import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
//DBCollection SEAWEED_TOWNSHIP = MongoUtils.getCollectionByName("fangjia_base", "seaweed_township", "192.168.0.235", 27018)
|
||||
//def count = 0
|
||||
//List<List<String>> data = Collections.synchronizedList([])
|
||||
//data << ["ID","城市","区域","小区名","经度","纬度","街道"]
|
||||
//
|
||||
//CommonUtils.findAllCity().each {def city->
|
||||
// DbUtils.multiThreadingExecuteQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
// if(StringUtils.isBlank(item.township) && item.lng){
|
||||
// count++
|
||||
// def lng = item.lng/10000000000d
|
||||
// def lat = item.lat/10000000000d
|
||||
// def location = "${lng},${lat}"
|
||||
// JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
// def row = []
|
||||
// row << item._id.toString()
|
||||
// row << item.city
|
||||
// row << item.region
|
||||
// row << item.name
|
||||
// row << lng
|
||||
// row << lat
|
||||
// BasicDBObject dbo = new BasicDBObject()
|
||||
// dbo.id = item._id.toString()
|
||||
// dbo.city = item.city
|
||||
// dbo.region = item.region
|
||||
// dbo.name = item.name
|
||||
// dbo.lng = lng
|
||||
// dbo.lat = lat
|
||||
// if(resp.regeocode && resp.regeocode.addressComponent && resp.regeocode.addressComponent.township){
|
||||
// row << resp.regeocode.addressComponent.township
|
||||
// dbo.township = resp.regeocode.addressComponent.township
|
||||
// }
|
||||
// println(row)
|
||||
// data << row
|
||||
// SEAWEED_TOWNSHIP.insert(dbo)
|
||||
// }
|
||||
// })
|
||||
//}
|
||||
//ExcelUtils.write("D:\\街道补充数据.xlsx", data, "补充数据")
|
||||
|
||||
//ExcelUtils.modify("D:\\街道补充数据.xlsx",{List item ->
|
||||
// if(item["G"] && item["H"] == null){
|
||||
// DbUtils.seaweed.update(new BasicDBObject("_id", item["A"]), new BasicDBObject('$set', new BasicDBObject('township', item["G"])))
|
||||
// }
|
||||
// println(item)
|
||||
//})
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\Êý¾Ý.xlsx",{List item->
|
||||
// if(DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]))){
|
||||
// item["D"] = "´æÔÚ"
|
||||
// }else{
|
||||
// item["D"] = "²»´æÔÚ"
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\错误经纬度.xlsx",{List item->
|
||||
// def location = "${item["A"]},${item["B"]}"
|
||||
// JSONObject resp = GaoDeApiUtils.geocode(location)
|
||||
// if(resp.regeocode && resp.regeocode.formatted_address){
|
||||
// item['G'] = resp.regeocode.formatted_address
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
|
||||
//def count = 0
|
||||
//CommonUtils.findAllCity().each {def city->
|
||||
// DbUtils.executeQueryOnSeaweed(DbUtils.buildSeaweedQuery(city),{BasicDBObject item->
|
||||
// if(item.township){
|
||||
// count++
|
||||
// }
|
||||
// }, new BasicDBObject("township", 1))
|
||||
//}
|
||||
//println(count)
|
||||
|
||||
DBCollection DISTRICT_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat","mongo.db.fangjia.com")
|
||||
ProvinceData.getAllCity("¸£½¨").each {def city->
|
||||
int count = 0
|
||||
def s = new TreeMap()
|
||||
DISTRICT_STAT.find(new BasicDBObject("weekend", Date.parse("yyyy-MM-dd", "2020-04-19")).append("city", city)).each {BasicDBObject stat->
|
||||
count ++
|
||||
int credit = (stat.credit * 100)/10
|
||||
s[credit] = s.get(credit, 0) + 1
|
||||
}
|
||||
|
||||
def data = []
|
||||
data << city
|
||||
data << count
|
||||
for(int i=0; i <=10; i++){
|
||||
data << s.get(i, 0)
|
||||
}
|
||||
println(data.join(","))
|
||||
}
|
||||
ElasticUtils.executeSqlQuery("select count(*), p_date from seawater where p_date > 20210501 group by p_date",{
|
||||
println(it)
|
||||
})
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.fangjia.script.wenyan.test
|
||||
|
||||
import com.aliyuncs.DefaultAcsClient
|
||||
import com.aliyuncs.IAcsClient
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityRequest
|
||||
import com.aliyuncs.address_purification.model.v20191118.GetAddressSimilarityResponse
|
||||
import com.aliyuncs.profile.DefaultProfile
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
def strt(String addr){
|
||||
|
||||
}
|
||||
|
||||
|
||||
ExcelUtils.modify("D:\\nextcloud\\数据\\地址类服务测试\\建行地址相似度评级测试.xlsx",{List item->
|
||||
def city = item["A"]
|
||||
def address1 = item["B"]
|
||||
def address2 = item["C"]
|
||||
HttpRequest http = new HttpRequest()
|
||||
def params = new HashMap()
|
||||
params["city"] = city
|
||||
params["address_1"] = address1
|
||||
params["address_2"] = address2
|
||||
params["token"]= "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB"
|
||||
JSONObject json = http.getJsonContent("http://open.fangjia.com/address/similarity", params, "UTF-8")
|
||||
JSONObject result = json.result
|
||||
item["D"] = result?.level
|
||||
item["E"] = result?.formattedAddress_1
|
||||
item["F"] = result?.formattedAddress_2
|
||||
item["G"] = result?.isSameAddress
|
||||
DefaultProfile profile = DefaultProfile.getProfile(
|
||||
"cn-hangzhou",
|
||||
"LTAIdaj4eSO79gzz",
|
||||
"xw6EsvqydZ9HC6zAvIB3ruQqVHynxf");
|
||||
IAcsClient client = new DefaultAcsClient(profile);
|
||||
GetAddressSimilarityRequest request = new GetAddressSimilarityRequest();
|
||||
request.setServiceCode("addrp")
|
||||
request.setAppKey("Xcgtyhupey0h")
|
||||
request.setText("${address1};${address2}")
|
||||
GetAddressSimilarityResponse response = client.getAcsResponse(request)
|
||||
String data = response.getData()
|
||||
json = JSONObject.fromObject(data)
|
||||
item["H"] = json["sim_level"]["level"]
|
||||
item["I"] = json["sim_level"]["address_a"]
|
||||
item["J"] = json["sim_level"]["address_b"]
|
||||
item["K"] = json["sim_level"]["is_same"]
|
||||
println(json)
|
||||
})
|
||||
|
||||
Reference in New Issue
Block a user