diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index 582a71f23..4f59fe761 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -274,6 +274,7 @@ public class BaiDuApiUtils { } return json.getJSONArray("result").getJSONObject(0); }catch (Exception e){ + System.out.println("错误:"+result); e.printStackTrace(); } return null; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 97cba74f5..c2efd484b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -34,7 +34,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["宁波"]// CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -53,9 +53,9 @@ class BigdataCommunityBatch extends BaseBatch{ // FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false); } - def fileName = "/home/goojia/ccb/bigdata_community.txt" + def fileName = "/home/goojia/ccb/bigdata_community.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat" } def allDatas = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index b56aa4f6b..82766d351 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -14,6 +14,7 @@ import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -27,7 +28,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -69,9 +70,9 @@ class BigdataDealBatch extends BaseBatch{ // ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据") } - def fileName = "/home/goojia/ccb/bigdata_deal.txt" + def fileName = "/home/goojia/ccb/bigdata_deal.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat" } FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false); @@ -399,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{ def startDate = null; // if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd")) + startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd")) // }else{ // startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) // } @@ -407,16 +408,21 @@ class BigdataDealBatch extends BaseBatch{ def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) for(def i=0;i=startPrice && obj.price<=endPrice){ rets << obj; } @@ -466,7 +479,7 @@ class BigdataDealBatch extends BaseBatch{ q.put("city", city) q.put("weekend", weekend) def dbcursor = DIST_SELL_STAT.find(q) - def map = [:] + def map = new ConcurrentHashMap() for(def d : dbcursor){ map.put(d.region+"@"+d.district_name, d.avg_price) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 162503f04..b8af43278 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -73,8 +73,8 @@ class BigdataForsaleBatch extends BaseBatch{ index++ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) - def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731); + def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831); } } @@ -144,9 +144,9 @@ class BigdataForsaleBatch extends BaseBatch{ } latch.await() - def fileName = "/home/goojia/ccb/bigdata_forsale.txt" + def fileName = "/home/goojia/ccb/bigdata_forsale.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.dat" } println city + " 房源总量:"+retMap.size() @@ -424,7 +424,6 @@ class BigdataForsaleBatch extends BaseBatch{ if (area == null) { return false; } - if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) { return true; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy index f80fa70dc..8a6e106ca 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataPoiBatch.groovy @@ -33,37 +33,37 @@ class BigdataPoiBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def citys = CcbCommonUtils.citys; // def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx" -// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) -// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) + def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1) + def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1) // def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1) -// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) -// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) + def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1) + def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1) // def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1) - def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1) +// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1) def rets = [] - def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校") +// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") +// rets.addAll(rows1) + + def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") rets.addAll(rows1) -// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁") -// rets.addAll(rows1) -// -// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") -// rets.addAll(rows2) + def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交") + rets.addAll(rows2) // // def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院") // rets.addAll(rows3) -// -// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") -// rets.addAll(rows4) -// -// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") -// rets.addAll(rows5) + + def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行") + rets.addAll(rows4) + + def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务") + rets.addAll(rows5) // // def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校") // rets.addAll(rows6) - def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"] + def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"] def fileName = "/home/goojia/ccb/bigdata_poi.txt" if(CommonDatautils.isWindow()){ @@ -119,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{ void run() { try{ totalCount.addAndGet(1) - if(citys.contains(d['A'])){ + if(citys.contains(d['D'])){ def ret = addValue(d, cityCodeDataMap, cityDistMap, type) if(ret){ rets << ret @@ -219,6 +219,8 @@ class BigdataPoiBatch extends BaseBatch{ ret << "" ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << d['C'] + ret << d['F'] ret << "FJW" return ret; @@ -226,10 +228,10 @@ class BigdataPoiBatch extends BaseBatch{ def addValue(d, cityCodeDataMap, cityDistMap, type){ def ret = [] - def citycode = cityCodeDataMap.get(d['A']) + def citycode = cityCodeDataMap.get(d['D']) ret << citycode - def distCodeMap = cityDistMap.get(d['A']) - def region =RegionData.formatRegion(d['A'], d['B']) + def distCodeMap = cityDistMap.get(d['D']) + def region =RegionData.formatRegion(d['D'], d['E']) if(region){ @@ -238,13 +240,13 @@ class BigdataPoiBatch extends BaseBatch{ ret << distObj.district_code }else{ ret << "" - println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F'] + println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F'] } }else{ ret << "" } - ret << new ObjectId() - ret << d['C'] + ret << d['A'] + ret << d['F'] ret << type if(type=="地铁"){ @@ -258,15 +260,15 @@ class BigdataPoiBatch extends BaseBatch{ }else if(type =="丧葬服务"){ ret << "ob101" }else if(type =="学校"){ -// if(d['F'].contains("中学")){ + if(d['F'].contains("中学")){ ret << "ed138" -// }else if(d['F'].contains("小学")){ -// ret << "ed114" -// } + }else if(d['F'].contains("小学")){ + ret << "ed114" + } } if(d['I']){ - def lat = d['J'].toDouble() - def lng = d['I'].toDouble() + def lat = d['I'].toDouble() + def lng = d['H'].toDouble() def loactionMap = GeoUtils.gcjToGps84(lat, lng) if(loactionMap){ ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) @@ -282,6 +284,8 @@ class BigdataPoiBatch extends BaseBatch{ ret << "" ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + ret << "" + ret << "" ret << "FJW" return ret; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 933425df2..e80d7fd22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -174,6 +174,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); +// q.put("name", "美辰香醒") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) @@ -186,7 +187,7 @@ class CcbCommonUtils { def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000) def rets = [] for(def d : dbsour){ - if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){ + if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){ if(d.address){ d.address = d.address.replaceAll("\r|\n|\\s", ""); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy new file mode 100644 index 000000000..f340f72ab --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -0,0 +1,226 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.sql.Sql +import org.apache.lucene.queryparser.classic.QueryParser +import org.bson.types.ObjectId +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class DistrictDelete { + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new DistrictDelete() +// task.run() + task.gzRun(); + } + + def run(){ + //"泉州","深圳", "重庆", + def citys = ["广州"];//"广州", + + def delIndex = 0 + for(def city : citys){ + def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"]) + + checkDist(city, dists) + } + } + + def checkDist(city,dists){ + + def rets = Collections.synchronizedList(new ArrayList()); + def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd") + + def totalCount = new AtomicLong() + def delIndex = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + println city +" 小区量:"+dists.size() + + def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"] + for(def i=0;i + count = r.count + }); + return count + } + + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + return houses.size(); + } + + + static def findHouseCount(def city, def region, def name, def alias){ + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + def aliasStr = escapeExprSpecialWord(alias.join("','")) + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and district_name in ('${aliasStr}')") + sqlBuffer.append(" and (cat = 'rent' or cat='sell')") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def count = null + while (rs.next()){ + count = rs.getInt("count") + } + return count; + } + + public static String escapeExprSpecialWord(String keyword) { + return QueryParser.escape(keyword); + } + + def gzRun(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d-> + if(d['U'] && d['U'].toString().toDouble().toInteger()==1){ + def id = d['E'] + BasicDBObject q=new BasicDBObject('name_cat',1) + q.put("m_date", new Date()) + DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q)) + index++ + println index + } + },0) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy index 3b2c217fc..71657eea3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/NameProcess.groovy @@ -107,7 +107,7 @@ class NameProcess { } } if(name && !name.contains("路") && !name.contains("街") && !name.contains("大道") && !name.contains("巷")){ - def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(号|排)")) + def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(排)")) if(h1){ name = name.replaceAll(h1,"") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index b6d403059..38c00f546 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -24,6 +24,7 @@ class SeaweedDataMatchProcess { rMap.estate_type2 = obj.estate_type2 rMap.city = obj.city rMap.region = obj.region + rMap.alias = obj.alias rMap.name = obj.name rMap.lat = obj.lat/10000000000.0 rMap.lng = obj.lng/10000000000.0 @@ -43,9 +44,6 @@ class SeaweedDataMatchProcess { } } - - - // if(rMap && rMap.id && region.contains(rMap.region)) { // def obj = CommonDatautils.findDistrictById(rMap.id.toString()); // if (obj) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy new file mode 100644 index 000000000..fecb21729 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DateUtil +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.util.DateUtils +import com.fangjia.script.yinjihaun.util.DingDingMessageUtil + +/** + * 快照监控 + * + * @author shenjian + * @since 2021/8/27 + */ +@BatchOwner(Watcher.ShenJian) +class SnapshotMonitor extends BaseBatch { + + // 运维组 运维警报 + static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + + static void main(String[] args) { + runBatch(new SnapshotMonitor(), args) + } + + + @Override + void process(Object args) { + // /_snapshot/es_backup/weekly-snap-2021.05* + // 监控两个月的快照 + String[] months = [DateUtils.formatDateByPattern(new Date(), "yyyy.MM"), DateUtils.formatDateByPattern(DateUtil.addMonthToDate(new Date(), -1), "yyyy.MM")] + String urlPrefix = "http://192.168.0.206:9200/_snapshot/es_backup/weekly-snap-" + + String msg = ""; + for (String month : months) { + // 必须分开写,不能有空格 + String[] cmds = ["curl", "-XGET", urlPrefix + month + "*", "--user", "elastic:goojia123456"] + String result = execCurl(cmds) + JSONObject jsonObject = JSONObject.parse(result); + + if (jsonObject.get("snapshots") != null) { + com.alibaba.fastjson.JSONArray jsonArray = jsonObject.get("snapshots") + for (JSONObject obj : jsonArray) { + String state = obj.get("state") + if (!"SUCCESS".equals(state)) { + msg = "【ES集群快照异常】\n" + obj.get("failures").toString() + DingDingMessageUtil.sendTextMessage(access_token, msg) + return ; + } + } + } + } + } + + static String execCurl(String[] cmds) { + ProcessBuilder process = new ProcessBuilder(cmds); + Process p; + try { + p = process.start(); + BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())); + StringBuilder builder = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null) { + builder.append(line); + builder.append(System.getProperty("line.separator")); + } + return builder.toString(); + + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } +}