模型级联城市导出

This commit is contained in:
沈健
2021-04-13 12:34:04 +08:00
parent c0704a0bf3
commit ddf08d464d
10 changed files with 1142 additions and 0 deletions
@@ -0,0 +1,183 @@
package com.fangjia.script.temp.shenjian
import com.alibaba.excel.EasyExcel
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBCursor
import com.mongodb.DBObject
import java.util.concurrent.*
/**
* 成交详情导出
*
* @author shenjian* @since 2021/3/29
*/
class DealPriceDetailExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017)
static final Map retMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(10)
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州",
"合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台",
"徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁",
"南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华",
"昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城",
"芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州",
"连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口",
"西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"]
// def cities = ["常州"]
def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"]
List<String> head = new ArrayList<>()
for (String name: headArr) {
List<String> temp = new ArrayList<>()
temp.add(name)
head.add(temp)
}
CountDownLatch latch = new CountDownLatch(cities.size())
DBObject filter = new BasicDBObject()
filter.put("city", 1)
filter.put("region", 1)
filter.put("district_name", 1)
filter.put("total_price", 1)
filter.put("trade_date", 1)
filter.put("avg_price", 1)
filter.put("area", 1)
filter.put("direction", 1)
filter.put("fitment", 1)
filter.put("height", 1)
filter.put("floor", 1)
filter.put("unit_num", 1)
filter.put("house_num", 1)
filter.put("room_num", 1)
filter.put("hall", 1)
filter.put("room", 1)
filter.put("toilet", 1)
for (def city : cities) {
final currCity = city
executorService.execute(new Runnable() {
@Override
void run() {
try {
final def ret = new CopyOnWriteArrayList()
// 小区
def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, obj)
if (obj.alias) {
// 小区可能存在多个别名
for (def a : obj.alias) {
map.put(obj.region + a, obj)
}
}
}
}
//ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"]
DBObject query = new BasicDBObject();
query.put("city", currCity)
query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).count())
fangjiaConn.find(query, filter).batchSize(10000).each { tempObj ->
final obj = tempObj
districtExecutorService.execute(new Runnable() {
@Override
void run() {
try {
if (map.get(obj.region + obj.district_name)) {
def row = []
row[0] = currCity
row[1] = obj.region
row[2] = obj.district_name
row[3] = obj.total_price
row[4] = obj.avg_price
if (obj.trade_date) {
row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd")
}
row[6] = obj.area
row[7] = obj.direction
row[8] = obj.fitment
row[9] = obj.height
row[10] = obj.floor
row[11] = obj.unit_num
row[12] = obj.house_num
row[13] = obj.room_num
row[14] = obj.hall
row[15] = obj.room
row[16] = obj.toilet
def objOur = map.get(obj.region + obj.district_name)
calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row)
ret.add(row)
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
} catch (Exception ex) {
println("出现异常111111.。。。。。。。。。。。")
ex.printStackTrace()
} finally {
districtLatch.countDown()
}
}
})
}
districtLatch.await()
EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret);
//ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息")
println("当前处理城市" + currCity + " " + ret.size())
} catch (Exception ex) {
println("出现异常22222.。。。。。。。。。。。")
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
}
latch.await()
executorService.shutdown()
districtExecutorService.shutdown()
}
private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) {
// 最相近一周的该小区均价
Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7)
Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7)
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
dbObject.put("region", objOur.region)
dbObject.put("district_name", objOur.name)
// 不可用append,会被覆盖
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
Double weekendAvgPrice = 0
dbCursor.each { o ->
weekendAvgPrice = o.avg_price
}
if (weekendAvgPrice == 0) {
println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name)
row[17] = ""
row[18] = ""
} else {
row[17] = weekendAvgPrice
row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice))
}
}
}
@@ -0,0 +1,158 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCursor
import org.apache.commons.lang3.time.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class PPE15Export {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30)
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def cities = ["扬州"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
CountDownLatch latch = new CountDownLatch(cities.size())
for (def cityTemp : cities) {
final city = cityTemp
println("当前处理城市:" + city)
executorService.execute(new Runnable() {
@Override
void run() {
try {
AtomicInteger totalCount = new AtomicInteger(0)
AtomicInteger count = new AtomicInteger(0)
def row = []
row[0] = city
// 小区
def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, obj)
if (obj.alias) {
// 小区可能存在多个别名
for (def a : obj.alias) {
map.put(obj.region + a, obj)
}
}
}
}
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).count())
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).each { obj ->
districtExecutorService.execute(new Runnable() {
@Override
void run() {
try {
if (map.get(obj.region + obj.district_name)) {
def objOur = map.get(obj.region + obj.district_name)
Double avgPrice = obj.avg_price
Date tradeDate = obj.trade_date
// 最相近一周的该小区均价
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
dbObject.put("region", obj.region)
dbObject.put("district_name", objOur.name)
// 不可用append,会被覆盖
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
dbObject.put("region", obj.region)
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name)
// .append("weekend", new BasicDBObject('$lte', nextWeekDate))
// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
Double weekendAvgPrice = 0
dbCursor.each { o ->
weekendAvgPrice = o.avg_price
}
if (weekendAvgPrice == 0) {
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
} else {
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
if (ppe15) {
count.incrementAndGet()
}
}
totalCount.incrementAndGet()
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
} catch (Exception ex) {
ex.printStackTrace()
} finally {
districtLatch.countDown()
}
}
})
};
districtLatch.await()
row[6] = totalCount.get()
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
retMap.put(city, row)
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
}
}
})
};
latch.await()
for (def city: cities) {
if (retMap.get(city)) {
result << retMap.get(city)
} else {
def row = []
row[0] = city
row[6] = 0
row[7] = 0
result << row
}
}
ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
executorService.shutdown()
districtExecutorService.shutdown()
}
}
@@ -0,0 +1,130 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCursor
import org.apache.commons.lang3.time.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class PPE15ExportTest {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
static main(args) {
// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州",
// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
def cities = ["常州"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
for (def cityTemp : cities) {
final city = cityTemp
println("当前处理城市:" + city)
try {
AtomicInteger totalCount = new AtomicInteger(0)
AtomicInteger count = new AtomicInteger(0)
def row = []
row[0] = city
// 小区
final def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, obj)
if (obj.alias) {
for (def a : obj.alias) {
map.put(obj.region + a, obj)
}
}
}
}
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
int ppe15Count = 0
CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
.batchSize(10000).count())
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24"))))
.batchSize(10000).each { obj ->
try {
if (map.get(obj.region + obj.district_name)) {
def objOur = map.get(obj.region + obj.district_name)
Double avgPrice = obj.avg_price
Date tradeDate = obj.trade_date
// 最相近一周的该小区均价
Date nextWeekDate = DateUtils.addDays(tradeDate, 7)
Date firstWeekDate = DateUtils.addDays(tradeDate, -7)
com.mongodb.DBObject dbObject = new BasicDBObject("city", city)
dbObject.put("region", obj.region)
dbObject.put("district_name", objOur.name)
dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate))
dbObject.put("region", obj.region)
DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1)
Double weekendAvgPrice = 0
dbCursor.each { o ->
weekendAvgPrice = o.avg_price
}
if (weekendAvgPrice == 0) {
println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name)
} else {
boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15
if (ppe15) {
count.incrementAndGet()
}
}
totalCount.incrementAndGet()
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
} catch (Exception ex) {
ex.printStackTrace()
} finally {
districtLatch.countDown()
}
}
districtLatch.await()
row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)"
retMap.put(city, row)
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
}
};
for (def city: cities) {
if (retMap.get(city)) {
result << retMap.get(city)
} else {
result << 0
}
}
ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
}
}
@@ -0,0 +1,153 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
/**
* 上海房屋租金导出
*
* @author shenjian
* @since 2021/03/22
*/
class RentExport {
static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017)
static ExecutorService executorService = Executors.newFixedThreadPool(30)
static Map retMap = new ConcurrentHashMap()
static main(args) {
def result = []
result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"]
int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
.batchSize(10000).count()
AtomicInteger size = new AtomicInteger(count)
println("上海小区" + size.get())
CountDownLatch latch = new CountDownLatch(count)
DBUtils.seaweed.find(new BasicDBObject("city", "上海"),
new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1))
.batchSize(10000).each {obj ->
executorService.submit(new Runnable() {
@Override
void run() {
try {
DBObject query = new BasicDBObject()
query.append("city", "上海").append("district_name", obj.name).append("region", obj.region)
.append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22")))
.append("house_type", new BasicDBObject('$lt', 5))
DBObject filter = new BasicDBObject()
filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1)
.append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1)
buildResult(query, filter, result, obj.lng, obj.lat)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
latch.countDown()
println("*********************" + size.decrementAndGet())
}
}
})
}
latch.await()
result.addAll(retMap.values())
com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金");
executorService.shutdown()
}
static boolean buildResult (def query, def filter, def result, def lng, def lat) {
collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj ->
def row = []
row[0] = obj.city
row[1] = obj.region
row[2] = obj.district_name
row[3] = obj.house_type
row[4] = obj.avg_price
row[5] = obj.sample_enum
row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d)
row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d)
if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) {
// 判断价格是否合理
if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) {
retMap.put(obj.district_name + obj.region + obj.house_type, row)
println(retMap.size())
} else {
println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type)
}
}
}
}
static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) {
if (retMap.contains(district_name + region + "1")) {
printf("一居室" + retMap.get(district_name + region + "1")[4])
}
if (retMap.contains(district_name + region + "2")) {
printf("二居室" + retMap.get(district_name + region + "2")[4])
}
if (retMap.contains(district_name + region + "3")) {
printf("三居室" + retMap.get(district_name + region + "3")[4])
}
println("当前居室" + house_type + ", 价格为" + avg_price)
switch (house_type) {
case 2:
println(retMap.get(district_name + region + "1"))
if (retMap.get(district_name + region + "1")) {
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
break;
case 3:
if (retMap.get(district_name + region + "1")) {
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
if (retMap.get(district_name + region + "2")) {
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
break;
case 4:
if (retMap.get(district_name + region + "1")) {
double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4])
if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
if (retMap.get(district_name + region + "2")) {
double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4])
if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
if (retMap.get(district_name + region + "3")) {
double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4])
if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) {
return false
}
}
break;
}
return true
}
}
@@ -0,0 +1,92 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class TotalTestExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州",
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def cities = ["贵阳"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
//CountDownLatch latch = new CountDownLatch(cities.size())
for (def city : cities) {
try {
def row = []
row[0] = city
// 小区
def map = [:]
DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1))
.batchSize(10000).each {obj ->
if (obj.region != null) {
map.put(obj.region + obj.name, 1)
if (obj.alias) {
for (def a : obj.alias) {
map.put(obj.region + a, 1)
}
}
}
}
// 测试数据量(小区近一年成交量)
int tradeCount = 0
fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))))
.batchSize(10000).each { obj ->
if (map.get(obj.region + obj.district_name)) {
tradeCount++
} else {
println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name)
}
}
row[6] = tradeCount
// PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价)
//int ppe15 =
//row[7] =
result << row
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
//latch.countDown()
}
}
//latch.await()
ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1")
}
}
@@ -0,0 +1,91 @@
package com.fangjia.script.temp.shenjian
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.lixue.util.DBUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
/**
* 模型级联项目涉及城市
*
* @author shenjian
* @since 2021/3/23
*/
class ValueRateCityInfoExport {
// 小区均价集合
static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017)
// 小区成交集合
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017)
static Map retMap = new ConcurrentHashMap()
// 在此更改时间
static final def TRADE_DATE_START = "2020-03-28"
static main(args) {
def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州",
"合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台",
"徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁",
"南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华",
"昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城",
"芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州",
"连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口",
"西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"]
//def cities = ["重庆","郑州", "长沙", "北京"]
def result = []
result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"]
//CountDownLatch latch = new CountDownLatch(cities.size())
for (def city : cities) {
try {
def row = []
row[0] = city
// 小区数量
int seaweedCount = DBUtils.seaweed.count( new BasicDBObject("city", city).append("cat", "district").append("visible", 0))
// 可估值小区数量
int valuateCount = collection.find(new BasicDBObject("city", city)
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
).count()
// 有样本估值数量及占比
int haveSampleCount = collection.find(new BasicDBObject("city", city)
.append("value_type", 2)
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
).count()
// 无样本估值数量及占比
int notHaveSampleCount = valuateCount - haveSampleCount
// 可信度大于0.5的数量
int creditCount = collection.find(new BasicDBObject("city", city)
.append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))
.append("credit", new BasicDBObject('$gte', 0.5d))
).count()
row[1] = seaweedCount
row[2] = valuateCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(valuateCount * 100) / Double.valueOf(seaweedCount)) + "%)"
if (valuateCount != 0) {
row[3] = haveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(haveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)"
row[4] = notHaveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(notHaveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)"
row[5] = creditCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(creditCount * 100) / Double.valueOf(valuateCount)) + "%)"
}
result << row
println("当前处理城市" + city + " " + row)
} catch(Exception ex) {
ex.printStackTrace()
} finally {
//latch.countDown()
}
}
//latch.await()
ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.0-test.xlsx", result, "Sheet1")
}
}
@@ -0,0 +1,124 @@
package com.fangjia.script.temp.shenjian.es
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import org.apache.commons.lang3.time.DateUtils
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.*
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.reindex.DeleteByQueryRequest
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.builder.SearchSourceBuilder
import java.util.concurrent.atomic.AtomicLong
/**
* 拷贝1.31前四周数据至3.28日缺失的四周数据中
* 注意时间更新
*
* @author shenjian
* @since 2021/3/30
*/
class CopyDataForAnnPricePre {
static RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient()
static final long timestamp = (DateUtils.parseDate("2021-03-28", "yyyy-MM-dd").getTime() -
DateUtils.parseDate("2021-01-31", "yyyy-MM-dd").getTime())
static final int days = timestamp / 1000 / 3600 / 24
static void main(String[] args) {
// 验证数据是否正确写入
//String[] weeksNew= ["2021-03-07", "2021-03-14", "2021-03-21", "2021-03-28"]
String[] weeksNew= ["2021-03-07", "2021-01-31"]
for (String week : weeksNew) {
println(DateUtils.parseDate(week, "yyyy-MM-dd").getTime())
}
//String[] weeksNew=["2021-03-28"]
// ///////////////////// 删除数据临时逻辑
// // 先删除数据
// DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("ann_price_pre")
// deleteRequest.setQuery(QueryBuilders.termQuery("date", DateUtils.parseDate("2021-03-07", "yyyy-MM-dd").getTime()))
// // 批次大小
// deleteRequest.setBatchSize(1000);
// restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT)
// /////////////////////
checkNum(weeksNew)
BulkRequest request = new BulkRequest();
for (String week : weeksNew) {
AtomicLong count = new AtomicLong(0)
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.addDays(DateUtils.parseDate(week, "yyyy-MM-dd"), -days).getTime()))
.size(10000);
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
SearchRequest searchRequest = new SearchRequest("ann_price_pre")
.source(searchSourceBuilder)
.scroll(scroll); // 设置scroll间隔
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //通过发送初始搜索请求来初始化搜索上下文
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits()
write(count, searchHits, timestamp, request);
while (searchHits != null && searchHits.length > 0) { //通过循环调用搜索滚动api来检索所有搜索命中,直到没有文档返回
//处理返回的搜索结果
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); //创建一个新的搜索滚动请求,保存最后返回的滚动标识符和滚动间隔
scrollRequest.scroll(scroll);
searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
write(count, searchHits, timestamp, request);
}
if (count.get() > 0) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT)
println(week + "剩余批量操作成功:" + bulkResponse.status())
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); //完成滚动后,清除滚动上下文
clearScrollRequest.addScrollId(scrollId);
ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
boolean succeeded = clearScrollResponse.isSucceeded();
println("清除滚动句柄" + succeeded)
}
checkNum(weeksNew)
restHighLevelClient.close()
}
static write(def count, def searchHits, def timestamp, def request) {
for (SearchHit hit : searchHits) {
Map<String, Object> map = hit.getSourceAsMap();
map.put("date", Long.valueOf(map.get("date").toString()) + timestamp)
request.add(new IndexRequest("ann_price_pre").source(map).opType(DocWriteRequest.OpType.CREATE));
count.incrementAndGet()
if (count.get() >= 5000) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
println("批量操作成功:" + bulkResponse.status())
request = new BulkRequest();
count = new AtomicLong(0)
}
}
}
static checkNum(def weeksNew) {
// 验证数据是否正确写入
for (String week : weeksNew) {
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.parseDate(week, "yyyy-MM-dd").getTime()));
SearchRequest searchRequest = new SearchRequest("ann_price_pre")
searchRequest.source(searchSourceBuilder)
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
println(searchResponse.status().name() + " " + searchResponse.getHits().totalHits)
}
}
}
@@ -0,0 +1,98 @@
package com.fangjia.script.temp.shenjian.es
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
/**
* 由于快照恢复字段类型错误
* 同步es6.4该索引至es7.11,先写入mongo
*
* @author shenjian
* @since 2021/3/30
*/
class SyncDistrictPriceStat {
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017)
static void main(String[] args) {
QueryBuilder queryBuilder = QueryBuilders.matchAllQuery()
def districtPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder)
//def newDistrictPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder)
// RestHighLevelClient restHighLevelClient206 = getClient206()
// RestHighLevelClient restHighLevelClient = getClient()
// //def districtPriceStats
// GetResponse getResponse = getDocument(restHighLevelClient)
for (def map : districtPriceStats) {
DBObject dbObject = new BasicDBObject()
dbObject.put("badcase_count", map.badcase_count)
dbObject.put("city", map.city)
dbObject.put("city_grade", map.city_grade)
dbObject.put("credit_district_count", map.credit_district_count)
dbObject.put("date", map.date)
dbObject.put("district_count", map.district_count)
dbObject.put("price_district_count", map.price_district_count)
dbObject.put("score", map.score)
dbObject.put("vote_district", map.vote_district)
fangjiaConn.save(dbObject)
// // 创建索引请求
// IndexRequest request = new IndexRequest("district_price_stat")
// request.id(id).timeout(TimeValue.timeValueSeconds(1))
// request.source(map)
//
// // 将请求发出去,获取响应结果
// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
//
// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE
// System.out.println(indexResponse.toString()); //响应内容
}
//restHighLevelClient.close()
}
// static RestHighLevelClient getClient() {
// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.216",9200,"http"))
// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
// @Override
// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
// httpClientBuilder.disableAuthCaching();
// CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
// return httpClientBuilder;
// }
// }))
// return restHighLevelClient
// }
//
// static RestHighLevelClient getClient206() {
// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.206",9200,"http"))
// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
// @Override
// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
// httpClientBuilder.disableAuthCaching();
// CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
// return httpClientBuilder;
// }
// }))
// return restHighLevelClient
// }
//
// static GetResponse getDocument(RestHighLevelClient restHighLevelClient206) throws IOException {
// SearchRequest searchRequest = new SearchRequest("district_price_stat");
// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
// //searchSourceBuilder.query(QueryBuilders.termQuery("url", "test"));
// searchRequest.source(searchSourceBuilder);
// SearchResponse response = restHighLevelClient206.search(searchRequest, RequestOptions.DEFAULT);
// System.out.println(response.getSourceAsString()); //获取文档的内容
// return response
// }
}
@@ -0,0 +1,78 @@
package com.fangjia.script.temp.shenjian.es
import com.fangjia.core.util.MongoUtils
import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils
import com.mongodb.DBCollection
import org.elasticsearch.action.DocWriteRequest
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.reindex.DeleteByQueryRequest
import java.util.concurrent.atomic.AtomicLong
/**
* 由于快照恢复字段类型错误
* 同步es6.4该索引至es7.11
*
* @author shenjian
* @since 2021/3/30
*/
class SyncDistrictPriceStat2 {
static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017)
static void main(String[] args) {
RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient()
// 先删除数据
DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("district_price_stat")
deleteRequest.setQuery(QueryBuilders.matchAllQuery())
// 批次大小
deleteRequest.setBatchSize(1000);
restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT)
BulkRequest request = new BulkRequest();
AtomicLong count = new AtomicLong()
fangjiaConn.find().batchSize(10000).each {map ->
Map dbObject = new HashMap()
dbObject.put("badcase_count", map.badcase_count)
dbObject.put("city", map.city)
dbObject.put("city_grade", map.city_grade)
dbObject.put("credit_district_count", map.credit_district_count)
dbObject.put("date", map.date)
dbObject.put("district_count", map.district_count)
dbObject.put("price_district_count", map.price_district_count)
dbObject.put("score", map.score)
dbObject.put("vote_district", map.vote_district)
request.add(new IndexRequest("district_price_stat").source(dbObject).opType(DocWriteRequest.OpType.CREATE));
count.incrementAndGet()
// 每一千写入
if (count.get() >= 1000) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
request = new BulkRequest();
count = new AtomicLong(0)
}
// 单个插入文档
// // 创建索引请求
// IndexRequest request = new IndexRequest("district_price_stat")
// request.timeout(TimeValue.timeValueSeconds(1))
// request.source(dbObject)
//
// // 将请求发出去,获取响应结果
// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT);
// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE
// System.out.println(indexResponse.toString()); //响应内容
}
// 不足一千写入
if (count.get() > 0) {
BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT);
}
restHighLevelClient.close()
}
}
@@ -0,0 +1,35 @@
package com.fangjia.script.temp.shenjian.es.util
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
/**
* @author shenjian
* @since 2021/3/30
*/
class RestHighLevelClientUtils {
static final def HOST = "192.168.0.216"
static RestHighLevelClient getClient() {
RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(HOST,9200,"http"))
.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
httpClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpClientBuilder;
}
}))
return restHighLevelClient
}
}