模型级联城市导出
This commit is contained in:
+183
@@ -0,0 +1,183 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.alibaba.excel.EasyExcel
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.*
|
||||
|
||||
/**
|
||||
* 成交详情导出
|
||||
*
|
||||
* @author shenjian* @since 2021/3/29
|
||||
*/
|
||||
class DealPriceDetailExport {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017)
|
||||
static final Map retMap = new ConcurrentHashMap()
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(10)
|
||||
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
|
||||
// 在此更改时间
|
||||
static final def TRADE_DATE_START = "2020-03-28"
|
||||
|
||||
static main(args) {
|
||||
def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州",
|
||||
"合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台",
|
||||
"徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁",
|
||||
"南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华",
|
||||
"昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城",
|
||||
"芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州",
|
||||
"连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口",
|
||||
"西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"]
|
||||
// def cities = ["常州"]
|
||||
|
||||
def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"]
|
||||
List<String> head = new ArrayList<>()
|
||||
for (String name: headArr) {
|
||||
List<String> temp = new ArrayList<>()
|
||||
temp.add(name)
|
||||
head.add(temp)
|
||||
}
|
||||
|
||||
CountDownLatch latch = new CountDownLatch(cities.size())
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.put("city", 1)
|
||||
filter.put("region", 1)
|
||||
filter.put("district_name", 1)
|
||||
filter.put("total_price", 1)
|
||||
filter.put("trade_date", 1)
|
||||
filter.put("avg_price", 1)
|
||||
filter.put("area", 1)
|
||||
filter.put("direction", 1)
|
||||
filter.put("fitment", 1)
|
||||
filter.put("height", 1)
|
||||
filter.put("floor", 1)
|
||||
filter.put("unit_num", 1)
|
||||
filter.put("house_num", 1)
|
||||
filter.put("room_num", 1)
|
||||
filter.put("hall", 1)
|
||||
filter.put("room", 1)
|
||||
filter.put("toilet", 1)
|
||||
for (def city : cities) {
|
||||
final currCity = city
|
||||
executorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
final def ret = new CopyOnWriteArrayList()
|
||||
// 小区
|
||||
def map = [:]
|
||||
DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
if (obj.region != null) {
|
||||
map.put(obj.region + obj.name, obj)
|
||||
if (obj.alias) {
|
||||
// 小区可能存在多个别名
|
||||
for (def a : obj.alias) {
|
||||
map.put(obj.region + a, obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"]
|
||||
DBObject query = new BasicDBObject();
|
||||
query.put("city", currCity)
|
||||
query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))
|
||||
|
||||
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
|
||||
.batchSize(10000).count())
|
||||
fangjiaConn.find(query, filter).batchSize(10000).each { tempObj ->
|
||||
final obj = tempObj
|
||||
districtExecutorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
if (map.get(obj.region + obj.district_name)) {
|
||||
def row = []
|
||||
row[0] = currCity
|
||||
row[1] = obj.region
|
||||
row[2] = obj.district_name
|
||||
row[3] = obj.total_price
|
||||
row[4] = obj.avg_price
|
||||
if (obj.trade_date) {
|
||||
row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd")
|
||||
}
|
||||
row[6] = obj.area
|
||||
row[7] = obj.direction
|
||||
row[8] = obj.fitment
|
||||
row[9] = obj.height
|
||||
row[10] = obj.floor
|
||||
row[11] = obj.unit_num
|
||||
row[12] = obj.house_num
|
||||
row[13] = obj.room_num
|
||||
row[14] = obj.hall
|
||||
row[15] = obj.room
|
||||
row[16] = obj.toilet
|
||||
def objOur = map.get(obj.region + obj.district_name)
|
||||
calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row)
|
||||
ret.add(row)
|
||||
} else {
|
||||
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
println("出现异常111111.。。。。。。。。。。。")
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
districtLatch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
districtLatch.await()
|
||||
EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret);
|
||||
//ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息")
|
||||
println("当前处理城市" + currCity + " " + ret.size())
|
||||
} catch (Exception ex) {
|
||||
println("出现异常22222.。。。。。。。。。。。")
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
latch.await()
|
||||
executorService.shutdown()
|
||||
districtExecutorService.shutdown()
|
||||
}
|
||||
|
||||
private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) {
|
||||
// 最相近一周的该小区均价
|
||||
Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7)
|
||||
Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7)
|
||||
|
||||
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
|
||||
dbObject.put("region", objOur.region)
|
||||
dbObject.put("district_name", objOur.name)
|
||||
// 不可用append,会被覆盖
|
||||
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
|
||||
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
Double weekendAvgPrice = 0
|
||||
dbCursor.each { o ->
|
||||
weekendAvgPrice = o.avg_price
|
||||
}
|
||||
|
||||
if (weekendAvgPrice == 0) {
|
||||
println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name)
|
||||
row[17] = ""
|
||||
row[18] = ""
|
||||
} else {
|
||||
row[17] = weekendAvgPrice
|
||||
row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice))
|
||||
}
|
||||
}
|
||||
}
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCursor
|
||||
import org.apache.commons.lang3.time.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 模型级联项目涉及城市
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/23
|
||||
*/
|
||||
class PPE15Export {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
|
||||
// 在此更改时间
|
||||
static final def TRADE_DATE_START = "2020-03-28"
|
||||
|
||||
static main(args) {
|
||||
def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
|
||||
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
|
||||
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
|
||||
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
|
||||
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
|
||||
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
|
||||
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
|
||||
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
//def cities = ["扬州"]
|
||||
def result = []
|
||||
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
|
||||
|
||||
CountDownLatch latch = new CountDownLatch(cities.size())
|
||||
for (def cityTemp : cities) {
|
||||
final city = cityTemp
|
||||
println("当前处理城市:" + city)
|
||||
executorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
AtomicInteger totalCount = new AtomicInteger(0)
|
||||
AtomicInteger count = new AtomicInteger(0)
|
||||
def row = []
|
||||
row[0] = city
|
||||
// 小区
|
||||
def map = [:]
|
||||
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
if (obj.region != null) {
|
||||
map.put(obj.region + obj.name, obj)
|
||||
if (obj.alias) {
|
||||
// 小区可能存在多个别名
|
||||
for (def a : obj.alias) {
|
||||
map.put(obj.region + a, obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
|
||||
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
|
||||
.batchSize(10000).count())
|
||||
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
|
||||
.batchSize(10000).each { obj ->
|
||||
districtExecutorService.execute(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
if (map.get(obj.region + obj.district_name)) {
|
||||
def objOur = map.get(obj.region + obj.district_name)
|
||||
Double avgPrice = obj.avg_price
|
||||
Date tradeDate = obj.trade_date
|
||||
// 最相近一周的该小区均价
|
||||
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
|
||||
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
|
||||
|
||||
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
|
||||
dbObject.put("region", obj.region)
|
||||
dbObject.put("district_name", objOur.name)
|
||||
// 不可用append,会被覆盖
|
||||
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
|
||||
dbObject.put("region", obj.region)
|
||||
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
|
||||
// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name)
|
||||
// .append("weekend", new BasicDBObject('$lte', nextWeekDate))
|
||||
// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
Double weekendAvgPrice = 0
|
||||
dbCursor.each { o ->
|
||||
weekendAvgPrice = o.avg_price
|
||||
}
|
||||
|
||||
if (weekendAvgPrice == 0) {
|
||||
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
|
||||
} else {
|
||||
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
|
||||
if (ppe15) {
|
||||
count.incrementAndGet()
|
||||
}
|
||||
}
|
||||
totalCount.incrementAndGet()
|
||||
} else {
|
||||
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
districtLatch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
districtLatch.await()
|
||||
row[6] = totalCount.get()
|
||||
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
|
||||
retMap.put(city, row)
|
||||
|
||||
println("当前处理城市" + city + " " + row)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
}
|
||||
}
|
||||
})
|
||||
};
|
||||
latch.await()
|
||||
for (def city: cities) {
|
||||
if (retMap.get(city)) {
|
||||
result << retMap.get(city)
|
||||
} else {
|
||||
def row = []
|
||||
row[0] = city
|
||||
row[6] = 0
|
||||
row[7] = 0
|
||||
result << row
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
|
||||
executorService.shutdown()
|
||||
districtExecutorService.shutdown()
|
||||
}
|
||||
}
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCursor
|
||||
import org.apache.commons.lang3.time.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 模型级联项目涉及城市
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/23
|
||||
*/
|
||||
class PPE15ExportTest {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
|
||||
static main(args) {
|
||||
// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
|
||||
// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
|
||||
// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
|
||||
// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
|
||||
// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
|
||||
// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
|
||||
// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
|
||||
// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
def cities = ["常州"]
|
||||
def result = []
|
||||
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
|
||||
|
||||
for (def cityTemp : cities) {
|
||||
final city = cityTemp
|
||||
println("当前处理城市:" + city)
|
||||
try {
|
||||
AtomicInteger totalCount = new AtomicInteger(0)
|
||||
AtomicInteger count = new AtomicInteger(0)
|
||||
def row = []
|
||||
row[0] = city
|
||||
// 小区
|
||||
final def map = [:]
|
||||
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
if (obj.region != null) {
|
||||
map.put(obj.region + obj.name, obj)
|
||||
if (obj.alias) {
|
||||
for (def a : obj.alias) {
|
||||
map.put(obj.region + a, obj)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
|
||||
int ppe15Count = 0
|
||||
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
|
||||
.batchSize(10000).count())
|
||||
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
|
||||
.batchSize(10000).each { obj ->
|
||||
try {
|
||||
if (map.get(obj.region + obj.district_name)) {
|
||||
def objOur = map.get(obj.region + obj.district_name)
|
||||
Double avgPrice = obj.avg_price
|
||||
Date tradeDate = obj.trade_date
|
||||
// 最相近一周的该小区均价
|
||||
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
|
||||
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
|
||||
|
||||
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
|
||||
dbObject.put("region", obj.region)
|
||||
dbObject.put("district_name", objOur.name)
|
||||
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
|
||||
dbObject.put("region", obj.region)
|
||||
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
|
||||
|
||||
Double weekendAvgPrice = 0
|
||||
dbCursor.each { o ->
|
||||
weekendAvgPrice = o.avg_price
|
||||
}
|
||||
|
||||
if (weekendAvgPrice == 0) {
|
||||
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
|
||||
} else {
|
||||
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
|
||||
if (ppe15) {
|
||||
count.incrementAndGet()
|
||||
}
|
||||
}
|
||||
totalCount.incrementAndGet()
|
||||
} else {
|
||||
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
districtLatch.countDown()
|
||||
}
|
||||
}
|
||||
districtLatch.await()
|
||||
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
|
||||
retMap.put(city, row)
|
||||
println("当前处理城市" + city + " " + row)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
|
||||
}
|
||||
};
|
||||
for (def city: cities) {
|
||||
if (retMap.get(city)) {
|
||||
result << retMap.get(city)
|
||||
} else {
|
||||
result << 0
|
||||
}
|
||||
}
|
||||
ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
|
||||
}
|
||||
}
|
||||
+153
@@ -0,0 +1,153 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
/**
|
||||
* 上海房屋租金导出
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/03/22
|
||||
*/
|
||||
class RentExport {
|
||||
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017)
|
||||
static ExecutorService executorService = Executors.newFixedThreadPool(30)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
|
||||
static main(args) {
|
||||
def result = []
|
||||
result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"]
|
||||
|
||||
int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
|
||||
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
|
||||
.batchSize(10000).count()
|
||||
AtomicInteger size = new AtomicInteger(count)
|
||||
println("上海小区" + size.get())
|
||||
CountDownLatch latch = new CountDownLatch(count)
|
||||
DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
|
||||
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try {
|
||||
DBObject query = new BasicDBObject()
|
||||
query.append("city", "上海").append("district_name", obj.name).append("region", obj.region)
|
||||
.append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22")))
|
||||
.append("house_type", new BasicDBObject('$lt', 5))
|
||||
|
||||
DBObject filter = new BasicDBObject()
|
||||
filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1)
|
||||
.append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1)
|
||||
|
||||
buildResult(query, filter, result, obj.lng, obj.lat)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
latch.countDown()
|
||||
println("*********************" + size.decrementAndGet())
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
latch.await()
|
||||
result.addAll(retMap.values())
|
||||
com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金");
|
||||
executorService.shutdown()
|
||||
}
|
||||
|
||||
static boolean buildResult (def query, def filter, def result, def lng, def lat) {
|
||||
collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj ->
|
||||
def row = []
|
||||
row[0] = obj.city
|
||||
row[1] = obj.region
|
||||
row[2] = obj.district_name
|
||||
row[3] = obj.house_type
|
||||
row[4] = obj.avg_price
|
||||
row[5] = obj.sample_enum
|
||||
row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d)
|
||||
row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d)
|
||||
|
||||
if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) {
|
||||
// 判断价格是否合理
|
||||
if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) {
|
||||
retMap.put(obj.district_name + obj.region + obj.house_type, row)
|
||||
println(retMap.size())
|
||||
} else {
|
||||
println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) {
|
||||
if (retMap.contains(district_name + region + "1")) {
|
||||
printf("一居室" + retMap.get(district_name + region + "1")[4])
|
||||
}
|
||||
if (retMap.contains(district_name + region + "2")) {
|
||||
printf("二居室" + retMap.get(district_name + region + "2")[4])
|
||||
}
|
||||
if (retMap.contains(district_name + region + "3")) {
|
||||
printf("三居室" + retMap.get(district_name + region + "3")[4])
|
||||
}
|
||||
println("当前居室" + house_type + ", 价格为" + avg_price)
|
||||
switch (house_type) {
|
||||
case 2:
|
||||
println(retMap.get(district_name + region + "1"))
|
||||
if (retMap.get(district_name + region + "1")) {
|
||||
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
|
||||
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 3:
|
||||
if (retMap.get(district_name + region + "1")) {
|
||||
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
|
||||
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (retMap.get(district_name + region + "2")) {
|
||||
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
|
||||
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
break;
|
||||
case 4:
|
||||
if (retMap.get(district_name + region + "1")) {
|
||||
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
|
||||
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (retMap.get(district_name + region + "2")) {
|
||||
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
|
||||
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
if (retMap.get(district_name + region + "3")) {
|
||||
double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4])
|
||||
if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
return true
|
||||
}
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
|
||||
/**
|
||||
* 模型级联项目涉及城市
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/23
|
||||
*/
|
||||
class TotalTestExport {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
// 在此更改时间
|
||||
static final def TRADE_DATE_START = "2020-03-28"
|
||||
|
||||
static main(args) {
|
||||
def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州",
|
||||
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
|
||||
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
|
||||
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
|
||||
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
|
||||
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
|
||||
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
|
||||
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
//def cities = ["贵阳"]
|
||||
|
||||
def result = []
|
||||
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
|
||||
|
||||
//CountDownLatch latch = new CountDownLatch(cities.size())
|
||||
for (def city : cities) {
|
||||
try {
|
||||
def row = []
|
||||
row[0] = city
|
||||
// 小区
|
||||
def map = [:]
|
||||
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
|
||||
.batchSize(10000).each {obj ->
|
||||
if (obj.region != null) {
|
||||
map.put(obj.region + obj.name, 1)
|
||||
if (obj.alias) {
|
||||
for (def a : obj.alias) {
|
||||
map.put(obj.region + a, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// 测试数据量(小区近一年成交量)
|
||||
int tradeCount = 0
|
||||
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
|
||||
.batchSize(10000).each { obj ->
|
||||
if (map.get(obj.region + obj.district_name)) {
|
||||
tradeCount++
|
||||
} else {
|
||||
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
|
||||
}
|
||||
}
|
||||
row[6] = tradeCount
|
||||
|
||||
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
|
||||
//int ppe15 =
|
||||
|
||||
//row[7] =
|
||||
|
||||
result << row
|
||||
println("当前处理城市" + city + " " + row)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
//latch.countDown()
|
||||
}
|
||||
}
|
||||
//latch.await()
|
||||
ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
|
||||
}
|
||||
}
|
||||
+91
@@ -0,0 +1,91 @@
|
||||
package com.fangjia.script.temp.shenjian
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.lixue.util.DBUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
|
||||
/**
|
||||
* 模型级联项目涉及城市
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/23
|
||||
*/
|
||||
class ValueRateCityInfoExport {
|
||||
|
||||
// 小区均价集合
|
||||
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
|
||||
// 小区成交集合
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
|
||||
static Map retMap = new ConcurrentHashMap()
|
||||
// 在此更改时间
|
||||
static final def TRADE_DATE_START = "2020-03-28"
|
||||
|
||||
static main(args) {
|
||||
def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州",
|
||||
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
|
||||
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
|
||||
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
|
||||
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
|
||||
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
|
||||
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
|
||||
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
|
||||
//def cities = ["重庆","郑州", "长沙", "北京"]
|
||||
|
||||
def result = []
|
||||
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
|
||||
|
||||
//CountDownLatch latch = new CountDownLatch(cities.size())
|
||||
for (def city : cities) {
|
||||
try {
|
||||
def row = []
|
||||
row[0] = city
|
||||
// 小区数量
|
||||
int seaweedCount = DBUtils.seaweed.count( new BasicDBObject("city", city).append("cat", "district").append("visible", 0))
|
||||
|
||||
// 可估值小区数量
|
||||
int valuateCount = collection.find(new BasicDBObject("city", city)
|
||||
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
|
||||
).count()
|
||||
|
||||
// 有样本估值数量及占比
|
||||
int haveSampleCount = collection.find(new BasicDBObject("city", city)
|
||||
.append("value_type", 2)
|
||||
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
|
||||
).count()
|
||||
|
||||
// 无样本估值数量及占比
|
||||
int notHaveSampleCount = valuateCount - haveSampleCount
|
||||
|
||||
// 可信度大于0.5的数量
|
||||
int creditCount = collection.find(new BasicDBObject("city", city)
|
||||
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
|
||||
.append("credit", new BasicDBObject('$gte', 0.5d))
|
||||
).count()
|
||||
|
||||
row[1] = seaweedCount
|
||||
row[2] = valuateCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(valuateCount * 100) / Double.valueOf(seaweedCount)) + "%)"
|
||||
if (valuateCount != 0) {
|
||||
row[3] = haveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(haveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)"
|
||||
row[4] = notHaveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(notHaveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)"
|
||||
row[5] = creditCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(creditCount * 100) / Double.valueOf(valuateCount)) + "%)"
|
||||
}
|
||||
|
||||
result << row
|
||||
println("当前处理城市" + city + " " + row)
|
||||
} catch(Exception ex) {
|
||||
ex.printStackTrace()
|
||||
} finally {
|
||||
//latch.countDown()
|
||||
}
|
||||
}
|
||||
//latch.await()
|
||||
ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.0-test.xlsx", result, "Sheet1")
|
||||
}
|
||||
}
|
||||
+124
@@ -0,0 +1,124 @@
|
||||
package com.fangjia.script.temp.shenjian.es
|
||||
|
||||
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
||||
import org.apache.commons.lang3.time.DateUtils
|
||||
import org.elasticsearch.action.DocWriteRequest
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.*
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.index.reindex.DeleteByQueryRequest
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 拷贝1.31前四周数据至3.28日缺失的四周数据中
|
||||
* 注意时间更新
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/30
|
||||
*/
|
||||
class CopyDataForAnnPricePre {
|
||||
|
||||
static RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient()
|
||||
|
||||
static final long timestamp = (DateUtils.parseDate("2021-03-28", "yyyy-MM-dd").getTime() -
|
||||
DateUtils.parseDate("2021-01-31", "yyyy-MM-dd").getTime())
|
||||
static final int days = timestamp / 1000 / 3600 / 24
|
||||
|
||||
static void main(String[] args) {
|
||||
// 验证数据是否正确写入
|
||||
//String[] weeksNew= ["2021-03-07", "2021-03-14", "2021-03-21", "2021-03-28"]
|
||||
String[] weeksNew= ["2021-03-07", "2021-01-31"]
|
||||
for (String week : weeksNew) {
|
||||
println(DateUtils.parseDate(week, "yyyy-MM-dd").getTime())
|
||||
}
|
||||
//String[] weeksNew=["2021-03-28"]
|
||||
// ///////////////////// 删除数据临时逻辑
|
||||
// // 先删除数据
|
||||
// DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("ann_price_pre")
|
||||
// deleteRequest.setQuery(QueryBuilders.termQuery("date", DateUtils.parseDate("2021-03-07", "yyyy-MM-dd").getTime()))
|
||||
// // 批次大小
|
||||
// deleteRequest.setBatchSize(1000);
|
||||
// restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT)
|
||||
// /////////////////////
|
||||
checkNum(weeksNew)
|
||||
|
||||
BulkRequest request = new BulkRequest();
|
||||
for (String week : weeksNew) {
|
||||
AtomicLong count = new AtomicLong(0)
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.addDays(DateUtils.parseDate(week, "yyyy-MM-dd"), -days).getTime()))
|
||||
.size(10000);
|
||||
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
|
||||
SearchRequest searchRequest = new SearchRequest("ann_price_pre")
|
||||
.source(searchSourceBuilder)
|
||||
.scroll(scroll); // 设置scroll间隔
|
||||
|
||||
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //通过发送初始搜索请求来初始化搜索上下文
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits()
|
||||
write(count, searchHits, timestamp, request);
|
||||
|
||||
while (searchHits != null && searchHits.length > 0) { //通过循环调用搜索滚动api来检索所有搜索命中,直到没有文档返回
|
||||
//处理返回的搜索结果
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); //创建一个新的搜索滚动请求,保存最后返回的滚动标识符和滚动间隔
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
write(count, searchHits, timestamp, request);
|
||||
}
|
||||
if (count.get() > 0) {
|
||||
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT)
|
||||
println(week + "剩余批量操作成功:" + bulkResponse.status())
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); //完成滚动后,清除滚动上下文
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
boolean succeeded = clearScrollResponse.isSucceeded();
|
||||
println("清除滚动句柄" + succeeded)
|
||||
}
|
||||
checkNum(weeksNew)
|
||||
|
||||
restHighLevelClient.close()
|
||||
}
|
||||
|
||||
static write(def count, def searchHits, def timestamp, def request) {
|
||||
for (SearchHit hit : searchHits) {
|
||||
Map<String, Object> map = hit.getSourceAsMap();
|
||||
map.put("date", Long.valueOf(map.get("date").toString()) + timestamp)
|
||||
|
||||
request.add(new IndexRequest("ann_price_pre").source(map).opType(DocWriteRequest.OpType.CREATE));
|
||||
count.incrementAndGet()
|
||||
if (count.get() >= 5000) {
|
||||
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
|
||||
println("批量操作成功:" + bulkResponse.status())
|
||||
request = new BulkRequest();
|
||||
count = new AtomicLong(0)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
static checkNum(def weeksNew) {
|
||||
// 验证数据是否正确写入
|
||||
for (String week : weeksNew) {
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.parseDate(week, "yyyy-MM-dd").getTime()));
|
||||
|
||||
SearchRequest searchRequest = new SearchRequest("ann_price_pre")
|
||||
searchRequest.source(searchSourceBuilder)
|
||||
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
|
||||
|
||||
println(searchResponse.status().name() + " " + searchResponse.getHits().totalHits)
|
||||
}
|
||||
}
|
||||
}
|
||||
+98
@@ -0,0 +1,98 @@
|
||||
package com.fangjia.script.temp.shenjian.es
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
/**
|
||||
* 由于快照恢复字段类型错误
|
||||
* 同步es6.4该索引至es7.11,先写入mongo
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/30
|
||||
*/
|
||||
class SyncDistrictPriceStat {
|
||||
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017)
|
||||
|
||||
static void main(String[] args) {
|
||||
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery()
|
||||
def districtPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder)
|
||||
//def newDistrictPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder)
|
||||
// RestHighLevelClient restHighLevelClient206 = getClient206()
|
||||
// RestHighLevelClient restHighLevelClient = getClient()
|
||||
// //def districtPriceStats
|
||||
// GetResponse getResponse = getDocument(restHighLevelClient)
|
||||
for (def map : districtPriceStats) {
|
||||
DBObject dbObject = new BasicDBObject()
|
||||
|
||||
dbObject.put("badcase_count", map.badcase_count)
|
||||
dbObject.put("city", map.city)
|
||||
dbObject.put("city_grade", map.city_grade)
|
||||
dbObject.put("credit_district_count", map.credit_district_count)
|
||||
dbObject.put("date", map.date)
|
||||
dbObject.put("district_count", map.district_count)
|
||||
dbObject.put("price_district_count", map.price_district_count)
|
||||
dbObject.put("score", map.score)
|
||||
dbObject.put("vote_district", map.vote_district)
|
||||
|
||||
fangjiaConn.save(dbObject)
|
||||
|
||||
// // 创建索引请求
|
||||
// IndexRequest request = new IndexRequest("district_price_stat")
|
||||
// request.id(id).timeout(TimeValue.timeValueSeconds(1))
|
||||
// request.source(map)
|
||||
//
|
||||
// // 将请求发出去,获取响应结果
|
||||
// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
|
||||
//
|
||||
// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE
|
||||
// System.out.println(indexResponse.toString()); //响应内容
|
||||
}
|
||||
//restHighLevelClient.close()
|
||||
}
|
||||
|
||||
// static RestHighLevelClient getClient() {
|
||||
// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.216",9200,"http"))
|
||||
// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
// @Override
|
||||
// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
|
||||
// httpClientBuilder.disableAuthCaching();
|
||||
// CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
// return httpClientBuilder;
|
||||
// }
|
||||
// }))
|
||||
// return restHighLevelClient
|
||||
// }
|
||||
//
|
||||
// static RestHighLevelClient getClient206() {
|
||||
// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.206",9200,"http"))
|
||||
// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
// @Override
|
||||
// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
|
||||
// httpClientBuilder.disableAuthCaching();
|
||||
// CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
// return httpClientBuilder;
|
||||
// }
|
||||
// }))
|
||||
// return restHighLevelClient
|
||||
// }
|
||||
//
|
||||
// static GetResponse getDocument(RestHighLevelClient restHighLevelClient206) throws IOException {
|
||||
// SearchRequest searchRequest = new SearchRequest("district_price_stat");
|
||||
// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
// //searchSourceBuilder.query(QueryBuilders.termQuery("url", "test"));
|
||||
// searchRequest.source(searchSourceBuilder);
|
||||
// SearchResponse response = restHighLevelClient206.search(searchRequest, RequestOptions.DEFAULT);
|
||||
// System.out.println(response.getSourceAsString()); //获取文档的内容
|
||||
// return response
|
||||
// }
|
||||
}
|
||||
+78
@@ -0,0 +1,78 @@
|
||||
package com.fangjia.script.temp.shenjian.es
|
||||
|
||||
import com.fangjia.core.util.MongoUtils
|
||||
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.action.DocWriteRequest
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.index.reindex.DeleteByQueryRequest
|
||||
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
/**
|
||||
* 由于快照恢复字段类型错误
|
||||
* 同步es6.4该索引至es7.11
|
||||
*
|
||||
* @author shenjian
|
||||
* @since 2021/3/30
|
||||
*/
|
||||
class SyncDistrictPriceStat2 {
|
||||
|
||||
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017)
|
||||
|
||||
static void main(String[] args) {
|
||||
RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient()
|
||||
// 先删除数据
|
||||
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("district_price_stat")
|
||||
deleteRequest.setQuery(QueryBuilders.matchAllQuery())
|
||||
// 批次大小
|
||||
deleteRequest.setBatchSize(1000);
|
||||
restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT)
|
||||
|
||||
BulkRequest request = new BulkRequest();
|
||||
AtomicLong count = new AtomicLong()
|
||||
fangjiaConn.find().batchSize(10000).each {map ->
|
||||
Map dbObject = new HashMap()
|
||||
dbObject.put("badcase_count", map.badcase_count)
|
||||
dbObject.put("city", map.city)
|
||||
dbObject.put("city_grade", map.city_grade)
|
||||
dbObject.put("credit_district_count", map.credit_district_count)
|
||||
dbObject.put("date", map.date)
|
||||
dbObject.put("district_count", map.district_count)
|
||||
dbObject.put("price_district_count", map.price_district_count)
|
||||
dbObject.put("score", map.score)
|
||||
dbObject.put("vote_district", map.vote_district)
|
||||
|
||||
request.add(new IndexRequest("district_price_stat").source(dbObject).opType(DocWriteRequest.OpType.CREATE));
|
||||
count.incrementAndGet()
|
||||
// 每一千写入
|
||||
if (count.get() >= 1000) {
|
||||
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
|
||||
request = new BulkRequest();
|
||||
count = new AtomicLong(0)
|
||||
}
|
||||
|
||||
// 单个插入文档
|
||||
// // 创建索引请求
|
||||
// IndexRequest request = new IndexRequest("district_price_stat")
|
||||
// request.timeout(TimeValue.timeValueSeconds(1))
|
||||
// request.source(dbObject)
|
||||
//
|
||||
// // 将请求发出去,获取响应结果
|
||||
// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
|
||||
|
||||
// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE
|
||||
// System.out.println(indexResponse.toString()); //响应内容
|
||||
}
|
||||
// 不足一千写入
|
||||
if (count.get() > 0) {
|
||||
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
|
||||
}
|
||||
restHighLevelClient.close()
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
package com.fangjia.script.temp.shenjian.es.util
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
|
||||
/**
|
||||
* @author shenjian
|
||||
* @since 2021/3/30
|
||||
*/
|
||||
class RestHighLevelClientUtils {
|
||||
|
||||
static final def HOST = "192.168.0.216"
|
||||
|
||||
static RestHighLevelClient getClient() {
|
||||
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(HOST,9200,"http"))
|
||||
.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
|
||||
httpClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpClientBuilder;
|
||||
}
|
||||
}))
|
||||
return restHighLevelClient
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user