Merge branch 'master' of http://gitlab.fangjia.com/dev0/main into open_gj
This commit is contained in:
@@ -274,6 +274,7 @@ public class BaiDuApiUtils {
|
||||
}
|
||||
return json.getJSONArray("result").getJSONObject(0);
|
||||
}catch (Exception e){
|
||||
System.out.println("错误:"+result);
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
|
||||
+3
-3
@@ -34,7 +34,7 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
|
||||
@Override
|
||||
public void process(Object args) {
|
||||
def citys = ["宁波"]// CcbCommonUtils.citys;
|
||||
def citys = CcbCommonUtils.citys;
|
||||
def rows = []
|
||||
|
||||
def headers = ["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途",
|
||||
@@ -53,9 +53,9 @@ class BigdataCommunityBatch extends BaseBatch{
|
||||
// FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community_${city}.txt",headers, "utf-8", false);
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_community.txt"
|
||||
def fileName = "/home/goojia/ccb/bigdata_community.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.txt"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_community.dat"
|
||||
}
|
||||
|
||||
def allDatas = []
|
||||
|
||||
+26
-13
@@ -14,6 +14,7 @@ import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
@@ -27,7 +28,7 @@ import java.util.regex.Pattern
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def curDate = 20210601//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
def curDate = 20210401//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd"));
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"]
|
||||
|
||||
@@ -69,9 +70,9 @@ class BigdataDealBatch extends BaseBatch{
|
||||
// ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\bigdata_deal_${city}.xlsx", cityDatas, "数据")
|
||||
}
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_deal.txt"
|
||||
def fileName = "/home/goojia/ccb/bigdata_deal.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.txt"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_deal.dat"
|
||||
}
|
||||
|
||||
FileUtils.batchCreateFile(rows, fileName,headers, "utf-8", false);
|
||||
@@ -399,7 +400,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def startDate = null;
|
||||
// if(citys.contains(city)){
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210501", "yyyyMMdd"))
|
||||
startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210401", "yyyyMMdd"))
|
||||
// }else{
|
||||
// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))
|
||||
// }
|
||||
@@ -407,16 +408,21 @@ class BigdataDealBatch extends BaseBatch{
|
||||
def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210701", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date())
|
||||
for(def i=0;i<dists.size();i++){
|
||||
final def dist = dists[i]
|
||||
// fixedThreadPool.execute(new Runnable() {
|
||||
// void run() {
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
def price = statMap.get(dist.region+"@"+dist.name)
|
||||
def startPrice = null
|
||||
def endPrice = null
|
||||
if(price){
|
||||
startPrice = (int)(price - price*0.5)
|
||||
endPrice = (int)(price*0.5 + price)
|
||||
if(dist.city=="上海"){
|
||||
startPrice = (int)(price - price*0.1)
|
||||
endPrice = (int)(price*0.5 + price)
|
||||
}else{
|
||||
startPrice = 1//(int)(price - price*0.5)
|
||||
endPrice = 500000//(int)(price*0.5 + price)
|
||||
}
|
||||
}
|
||||
def trades = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, startPrice, endPrice)
|
||||
if(trades){
|
||||
@@ -428,8 +434,8 @@ class BigdataDealBatch extends BaseBatch{
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
// };
|
||||
// })
|
||||
};
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
return tradesList
|
||||
@@ -447,8 +453,15 @@ class BigdataDealBatch extends BaseBatch{
|
||||
|
||||
def price = statMap.get(obj.fjRegion+"@"+obj.fjName)
|
||||
if(price){
|
||||
def startPrice = (int)(price - price*0.5)
|
||||
def endPrice= (int)(price*0.5 + price)
|
||||
def startPrice = 0
|
||||
def endPrice= 0
|
||||
if(obj.fjCity=="上海"){
|
||||
startPrice = (int)(price - price*0.1)
|
||||
endPrice = (int)(price*0.5 + price)
|
||||
}else{
|
||||
startPrice = 1//(int)(price - price*0.5)
|
||||
endPrice = 500000//(int)(price*0.5 + price)
|
||||
}
|
||||
if(obj.price>=startPrice && obj.price<=endPrice){
|
||||
rets << obj;
|
||||
}
|
||||
@@ -466,7 +479,7 @@ class BigdataDealBatch extends BaseBatch{
|
||||
q.put("city", city)
|
||||
q.put("weekend", weekend)
|
||||
def dbcursor = DIST_SELL_STAT.find(q)
|
||||
def map = [:]
|
||||
def map = new ConcurrentHashMap()
|
||||
for(def d : dbcursor){
|
||||
map.put(d.region+"@"+d.district_name, d.avg_price)
|
||||
}
|
||||
|
||||
+4
-5
@@ -73,8 +73,8 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
index++
|
||||
// def distIds = getDistrictIds(cityCodeDataMap.get(city))
|
||||
def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city)
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "estate_type2", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210731);
|
||||
def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]);
|
||||
exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20200701, 20210831);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -144,9 +144,9 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
}
|
||||
latch.await()
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_forsale.txt"
|
||||
def fileName = "/home/goojia/ccb/bigdata_forsale.dat"
|
||||
if(CommonDatautils.isWindow()){
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.txt"
|
||||
fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_forsale.dat"
|
||||
}
|
||||
|
||||
println city + " 房源总量:"+retMap.size()
|
||||
@@ -424,7 +424,6 @@ class BigdataForsaleBatch extends BaseBatch{
|
||||
if (area == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (area >= 150 && StringUtils.indexOfAny(name, TAGS) > -1) {
|
||||
return true;
|
||||
}
|
||||
|
||||
+35
-31
@@ -33,37 +33,37 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
def cityCodeDataMap = CcbCommonUtils.findCityCodeValue()
|
||||
def citys = CcbCommonUtils.citys;
|
||||
// def path = "D:\\数据\\建行\\房产估值\\poi\\公交.xlsx"
|
||||
// def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
// def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
def subwaydatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\地铁.xlsx", 0,1)
|
||||
def busDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\公交.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\医院.xlsx", 0,1)
|
||||
// def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
// def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
def bankDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\银行.xlsx", 0,1)
|
||||
def dealDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\丧葬服务.xlsx", 0,1)
|
||||
// def schoolDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\学校.xlsx", 0,1)
|
||||
def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\中学.xlsx", 0,1)
|
||||
// def hosptialDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\poi\\八城市医院(二次审核).xlsx", 0,1)
|
||||
def rets = []
|
||||
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "学校")
|
||||
// def rows1 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows1)
|
||||
|
||||
def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
rets.addAll(rows1)
|
||||
|
||||
// def rows1 = createDatas(citys, cityCodeDataMap, subwaydatas, "地铁")
|
||||
// rets.addAll(rows1)
|
||||
//
|
||||
// def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
// rets.addAll(rows2)
|
||||
def rows2 = createDatas(citys, cityCodeDataMap, busDatas, "公交")
|
||||
rets.addAll(rows2)
|
||||
//
|
||||
// def rows3 = createDatas(citys, cityCodeDataMap, hosptialDatas, "医院")
|
||||
// rets.addAll(rows3)
|
||||
//
|
||||
// def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
// rets.addAll(rows4)
|
||||
//
|
||||
// def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
// rets.addAll(rows5)
|
||||
|
||||
def rows4 = createDatas(citys, cityCodeDataMap, bankDatas, "银行")
|
||||
rets.addAll(rows4)
|
||||
|
||||
def rows5 = createDatas(citys, cityCodeDataMap, dealDatas, "丧葬服务")
|
||||
rets.addAll(rows5)
|
||||
//
|
||||
// def rows6 = createDatas(citys, cityCodeDataMap, schoolDatas, "学校")
|
||||
// rets.addAll(rows6)
|
||||
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","来源公司"]
|
||||
def headers = ["城市ID","行政区ID","ID","Poi名称","类型","类型编码","纬度","经度","备注","创建时间","更新时间","性质","级别","来源公司"]
|
||||
|
||||
def fileName = "/home/goojia/ccb/bigdata_poi.txt"
|
||||
if(CommonDatautils.isWindow()){
|
||||
@@ -119,7 +119,7 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
void run() {
|
||||
try{
|
||||
totalCount.addAndGet(1)
|
||||
if(citys.contains(d['A'])){
|
||||
if(citys.contains(d['D'])){
|
||||
def ret = addValue(d, cityCodeDataMap, cityDistMap, type)
|
||||
if(ret){
|
||||
rets << ret
|
||||
@@ -219,6 +219,8 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << d['C']
|
||||
ret << d['F']
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
@@ -226,10 +228,10 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
|
||||
def addValue(d, cityCodeDataMap, cityDistMap, type){
|
||||
def ret = []
|
||||
def citycode = cityCodeDataMap.get(d['A'])
|
||||
def citycode = cityCodeDataMap.get(d['D'])
|
||||
ret << citycode
|
||||
def distCodeMap = cityDistMap.get(d['A'])
|
||||
def region =RegionData.formatRegion(d['A'], d['B'])
|
||||
def distCodeMap = cityDistMap.get(d['D'])
|
||||
def region =RegionData.formatRegion(d['D'], d['E'])
|
||||
|
||||
if(region){
|
||||
|
||||
@@ -238,13 +240,13 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
ret << distObj.district_code
|
||||
}else{
|
||||
ret << ""
|
||||
println d['A']+" " + d['B']+" 无对应的区域编码 "+d['F']
|
||||
println d['D']+" " + d['E']+" 无对应的区域编码 "+d['F']
|
||||
}
|
||||
}else{
|
||||
ret << ""
|
||||
}
|
||||
ret << new ObjectId()
|
||||
ret << d['C']
|
||||
ret << d['A']
|
||||
ret << d['F']
|
||||
ret << type
|
||||
|
||||
if(type=="地铁"){
|
||||
@@ -258,15 +260,15 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
}else if(type =="丧葬服务"){
|
||||
ret << "ob101"
|
||||
}else if(type =="学校"){
|
||||
// if(d['F'].contains("中学")){
|
||||
if(d['F'].contains("中学")){
|
||||
ret << "ed138"
|
||||
// }else if(d['F'].contains("小学")){
|
||||
// ret << "ed114"
|
||||
// }
|
||||
}else if(d['F'].contains("小学")){
|
||||
ret << "ed114"
|
||||
}
|
||||
}
|
||||
if(d['I']){
|
||||
def lat = d['J'].toDouble()
|
||||
def lng = d['I'].toDouble()
|
||||
def lat = d['I'].toDouble()
|
||||
def lng = d['H'].toDouble()
|
||||
def loactionMap = GeoUtils.gcjToGps84(lat, lng)
|
||||
if(loactionMap){
|
||||
ret << com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6)
|
||||
@@ -282,6 +284,8 @@ class BigdataPoiBatch extends BaseBatch{
|
||||
ret << ""
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss")
|
||||
ret << ""
|
||||
ret << ""
|
||||
ret << "FJW"
|
||||
|
||||
return ret;
|
||||
|
||||
+2
-1
@@ -174,6 +174,7 @@ class CcbCommonUtils {
|
||||
q.put("visible",0)
|
||||
q.put("cat", cat);
|
||||
q.put("city", city);
|
||||
// q.put("name", "美辰香醒")
|
||||
// q.put("name_cat", 0);
|
||||
// q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"]))
|
||||
|
||||
@@ -186,7 +187,7 @@ class CcbCommonUtils {
|
||||
def dbsour = getDataStoreConnection().find(q, keys).batchSize(2000)
|
||||
def rets = []
|
||||
for(def d : dbsour){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2)){
|
||||
if(d.name_cat == 0 && d.lng && ["普通住宅","商住"].contains(d.estate_type2) && !d.name.contains("洋房") && !d.name.contains("公房")){
|
||||
if(d.address){
|
||||
d.address = d.address.replaceAll("\r|\n|\\s", "");
|
||||
}
|
||||
|
||||
+226
@@ -0,0 +1,226 @@
|
||||
package com.fangjia.script.ccbevaluation
|
||||
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.Sql
|
||||
import org.apache.lucene.queryparser.classic.QueryParser
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
class DistrictDelete {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30)
|
||||
|
||||
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
dataSource.setProperties(connectionProperties)
|
||||
}
|
||||
|
||||
def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl",
|
||||
"GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver")
|
||||
|
||||
static main(args) {
|
||||
def task = new DistrictDelete()
|
||||
// task.run()
|
||||
task.gzRun();
|
||||
}
|
||||
|
||||
def run(){
|
||||
//"泉州","深圳", "重庆",
|
||||
def citys = ["广州"];//"广州",
|
||||
|
||||
def delIndex = 0
|
||||
for(def city : citys){
|
||||
def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"])
|
||||
|
||||
checkDist(city, dists)
|
||||
}
|
||||
}
|
||||
|
||||
def checkDist(city,dists){
|
||||
|
||||
def rets = Collections.synchronizedList(new ArrayList());
|
||||
def startDate = DateUtils.parseFromTightByPattern("2015-01-01","yyyy-MM-dd")
|
||||
def endDate = DateUtils.parseFromTightByPattern("2021-12-31","yyyy-MM-dd")
|
||||
|
||||
def totalCount = new AtomicLong()
|
||||
def delIndex = new AtomicLong()
|
||||
final CountDownLatch latch = new CountDownLatch(dists.size())
|
||||
println city +" 小区量:"+dists.size()
|
||||
|
||||
def keys = ["大厦","宿舍","宅","商住","广场","教师","工作站","学院", "SOLO", "时尚","大学","学员","职工","SOHO"]
|
||||
for(def i=0;i<dists.size();i++) {
|
||||
final def dist = dists[i];
|
||||
fixedThreadPool.execute(new Runnable() {
|
||||
void run() {
|
||||
try {
|
||||
totalCount.addAndGet(1)
|
||||
// def count = findHouses(city, dist.region, "sell", 20190101,20211231, dist.name, dist.alias)
|
||||
def count = findHouseCount(city, dist.region, dist.name, dist.alias)
|
||||
def tradeCount = 0
|
||||
if(!count){
|
||||
tradeCount = CcbCommonUtils.findTradesDetail(dist, startDate, endDate, 0, 300000)
|
||||
if(!tradeCount){
|
||||
def nameFlag = false
|
||||
for(def key : keys){
|
||||
if(dist.name.contains(key)){
|
||||
nameFlag = true
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(!nameFlag){
|
||||
if(dist.name.size()<=3 && (dist.name.contains("村") || dist.name.contains("楼") || dist.name.contains("居") )){
|
||||
nameFlag = true
|
||||
}
|
||||
}
|
||||
def fieldFlag = false
|
||||
if(dist.finish_building_date && dist.build_type){
|
||||
fieldFlag = true
|
||||
}
|
||||
|
||||
if(!tradeCount && !count && !fieldFlag){
|
||||
def rowCount = findCount(dist._id.toString())
|
||||
if(rowCount || nameFlag){
|
||||
delIndex.addAndGet(1)
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id', dist._id),new BasicDBObject('$set',q))
|
||||
println "[ ${delIndex.get()}/${totalCount.get()} ] "+city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
|
||||
}else{
|
||||
println city +" "+dist.region + " "+dist.name + " house:"+count+" trade:"+tradeCount
|
||||
}
|
||||
|
||||
rets << [dist._id.toString(),dist.city, dist.region, dist.name, dist.address]
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
latch.await()
|
||||
|
||||
ExcelUtils.write("D:\\数据\\建行\\房产估值\\多余小区_${city}.xlsx",rets, "无房源")
|
||||
}
|
||||
|
||||
|
||||
def findCount(distId){
|
||||
def sql = "select count(*) as count from fjw_community_artifi_check where commid='${distId}'";
|
||||
def count = 0
|
||||
|
||||
sqlInstance.eachRow(sql,{r->
|
||||
count = r.count
|
||||
});
|
||||
return count
|
||||
}
|
||||
|
||||
|
||||
|
||||
def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){
|
||||
|
||||
QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false);
|
||||
QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false);
|
||||
|
||||
QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat))
|
||||
.must(date_range).must(area_range)
|
||||
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
if(alias && alias.size()>0){
|
||||
BoolQueryBuilder inBuilder = QueryBuilders.boolQuery();
|
||||
for(def s : alias){
|
||||
inBuilder.should(QueryBuilders.matchQuery("district_name", s));
|
||||
}
|
||||
queryBuilder.must(inBuilder)
|
||||
}
|
||||
def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder);
|
||||
return houses.size();
|
||||
}
|
||||
|
||||
|
||||
static def findHouseCount(def city, def region, def name, def alias){
|
||||
if(!alias){
|
||||
alias = []
|
||||
}
|
||||
if(!alias.contains(name)){
|
||||
alias.add(name)
|
||||
}
|
||||
if(name.contains("弄小区")){
|
||||
def newName = name.replace("弄小区","弄")
|
||||
if(!alias.contains(newName)){
|
||||
alias << newName
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
def aliasStr = escapeExprSpecialWord(alias.join("','"))
|
||||
final StringBuffer sqlBuffer = new StringBuffer()
|
||||
sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1")
|
||||
sqlBuffer.append(" and city = '"+city+"'")
|
||||
sqlBuffer.append(" and region = '"+region+"'")
|
||||
sqlBuffer.append(" and district_name in ('${aliasStr}')")
|
||||
sqlBuffer.append(" and (cat = 'rent' or cat='sell')")
|
||||
ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString())
|
||||
def count = null
|
||||
while (rs.next()){
|
||||
count = rs.getInt("count")
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
public static String escapeExprSpecialWord(String keyword) {
|
||||
return QueryParser.escape(keyword);
|
||||
}
|
||||
|
||||
def gzRun(){
|
||||
def index = 0
|
||||
ExcelUtils.modify("D:\\数据\\建行\\房产估值\\经纬度有问题的小区.xlsx",{d->
|
||||
if(d['U'] && d['U'].toString().toDouble().toInteger()==1){
|
||||
def id = d['E']
|
||||
BasicDBObject q=new BasicDBObject('name_cat',1)
|
||||
q.put("m_date", new Date())
|
||||
DbUtils.seaweed.update(new BasicDBObject('_id',new ObjectId(id)),new BasicDBObject('$set',q))
|
||||
index++
|
||||
println index
|
||||
}
|
||||
},0)
|
||||
}
|
||||
|
||||
}
|
||||
+1
-1
@@ -107,7 +107,7 @@ class NameProcess {
|
||||
}
|
||||
}
|
||||
if(name && !name.contains("路") && !name.contains("街") && !name.contains("大道") && !name.contains("巷")){
|
||||
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(号|排)"))
|
||||
def h1 = DataUtil.match(name, Pattern.compile("(\\w+)?-?\\w+(排)"))
|
||||
if(h1){
|
||||
name = name.replaceAll(h1,"")
|
||||
}
|
||||
|
||||
+1
-3
@@ -24,6 +24,7 @@ class SeaweedDataMatchProcess {
|
||||
rMap.estate_type2 = obj.estate_type2
|
||||
rMap.city = obj.city
|
||||
rMap.region = obj.region
|
||||
rMap.alias = obj.alias
|
||||
rMap.name = obj.name
|
||||
rMap.lat = obj.lat/10000000000.0
|
||||
rMap.lng = obj.lng/10000000000.0
|
||||
@@ -43,9 +44,6 @@ class SeaweedDataMatchProcess {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
// if(rMap && rMap.id && region.contains(rMap.region)) {
|
||||
// def obj = CommonDatautils.findDistrictById(rMap.id.toString());
|
||||
// if (obj) {
|
||||
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.yinjihaun.util.DingDingMessageUtil
|
||||
|
||||
/**
|
||||
* 快照监控
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/8/27
|
||||
*/
|
||||
@BatchOwner(Watcher.ShenJian)
|
||||
class SnapshotMonitor extends BaseBatch {
|
||||
|
||||
// 运维组 运维警报
|
||||
static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
static void main(String[] args) {
|
||||
runBatch(new SnapshotMonitor(), args)
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
void process(Object args) {
|
||||
// /_snapshot/es_backup/weekly-snap-2021.05*
|
||||
// 监控两个月的快照
|
||||
String[] months = [DateUtils.formatDateByPattern(new Date(), "yyyy.MM"), DateUtils.formatDateByPattern(DateUtil.addMonthToDate(new Date(), -1), "yyyy.MM")]
|
||||
String urlPrefix = "http://192.168.0.206:9200/_snapshot/es_backup/weekly-snap-"
|
||||
|
||||
String msg = "";
|
||||
for (String month : months) {
|
||||
// 必须分开写,不能有空格
|
||||
String[] cmds = ["curl", "-XGET", urlPrefix + month + "*", "--user", "elastic:goojia123456"]
|
||||
String result = execCurl(cmds)
|
||||
JSONObject jsonObject = JSONObject.parse(result);
|
||||
|
||||
if (jsonObject.get("snapshots") != null) {
|
||||
com.alibaba.fastjson.JSONArray jsonArray = jsonObject.get("snapshots")
|
||||
for (JSONObject obj : jsonArray) {
|
||||
String state = obj.get("state")
|
||||
if (!"SUCCESS".equals(state)) {
|
||||
msg = "【ES集群快照异常】\n" + obj.get("failures").toString()
|
||||
DingDingMessageUtil.sendTextMessage(access_token, msg)
|
||||
return ;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static String execCurl(String[] cmds) {
|
||||
ProcessBuilder process = new ProcessBuilder(cmds);
|
||||
Process p;
|
||||
try {
|
||||
p = process.start();
|
||||
BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream()));
|
||||
StringBuilder builder = new StringBuilder();
|
||||
String line = null;
|
||||
while ((line = reader.readLine()) != null) {
|
||||
builder.append(line);
|
||||
builder.append(System.getProperty("line.separator"));
|
||||
}
|
||||
return builder.toString();
|
||||
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user