From ddf08d464dabc5c81dea15ca20e5907307a58aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Apr 2021 12:34:04 +0800 Subject: [PATCH] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=BA=A7=E8=81=94=E5=9F=8E?= =?UTF-8?q?=E5=B8=82=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/DealPriceDetailExport.groovy | 183 ++++++++++++++++++ .../script/temp/shenjian/PPE15Export.groovy | 158 +++++++++++++++ .../temp/shenjian/PPE15ExportTest.groovy | 130 +++++++++++++ .../script/temp/shenjian/RentExport.groovy | 153 +++++++++++++++ .../temp/shenjian/TotalTestExport.groovy | 92 +++++++++ .../shenjian/ValueRateCityInfoExport.groovy | 91 +++++++++ .../shenjian/es/CopyDataForAnnPricePre.groovy | 124 ++++++++++++ .../shenjian/es/SyncDistrictPriceStat.groovy | 98 ++++++++++ .../shenjian/es/SyncDistrictPriceStat2.groovy | 78 ++++++++ .../es/util/RestHighLevelClientUtils.groovy | 35 ++++ 10 files changed, 1142 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy new file mode 100644 index 000000000..9f532247a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy @@ -0,0 +1,183 @@ +package com.fangjia.script.temp.shenjian + +import com.alibaba.excel.EasyExcel +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject + +import java.util.concurrent.* + +/** + * 成交详情导出 + * + * @author shenjian* @since 2021/3/29 + */ +class DealPriceDetailExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017) + static final Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(10) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州", + "合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台", + "徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁", + "南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华", + "昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城", + "芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州", + "连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口", + "西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"] +// def cities = ["常州"] + + def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + + CountDownLatch latch = new CountDownLatch(cities.size()) + DBObject filter = new BasicDBObject() + filter.put("city", 1) + filter.put("region", 1) + filter.put("district_name", 1) + filter.put("total_price", 1) + filter.put("trade_date", 1) + filter.put("avg_price", 1) + filter.put("area", 1) + filter.put("direction", 1) + filter.put("fitment", 1) + filter.put("height", 1) + filter.put("floor", 1) + filter.put("unit_num", 1) + filter.put("house_num", 1) + filter.put("room_num", 1) + filter.put("hall", 1) + filter.put("room", 1) + filter.put("toilet", 1) + for (def city : cities) { + final currCity = city + executorService.execute(new Runnable() { + @Override + void run() { + try { + final def ret = new CopyOnWriteArrayList() + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + //ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"] + DBObject query = new BasicDBObject(); + query.put("city", currCity) + query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))) + + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(query, filter).batchSize(10000).each { tempObj -> + final obj = tempObj + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def row = [] + row[0] = currCity + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.total_price + row[4] = obj.avg_price + if (obj.trade_date) { + row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd") + } + row[6] = obj.area + row[7] = obj.direction + row[8] = obj.fitment + row[9] = obj.height + row[10] = obj.floor + row[11] = obj.unit_num + row[12] = obj.house_num + row[13] = obj.room_num + row[14] = obj.hall + row[15] = obj.room + row[16] = obj.toilet + def objOur = map.get(obj.region + obj.district_name) + calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row) + ret.add(row) + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + println("出现异常111111.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + } + districtLatch.await() + EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret); + //ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息") + println("当前处理城市" + currCity + " " + ret.size()) + } catch (Exception ex) { + println("出现异常22222.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + executorService.shutdown() + districtExecutorService.shutdown() + } + + private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) { + // 最相近一周的该小区均价 + Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", objOur.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name) + row[17] = "" + row[18] = "" + } else { + row[17] = weekendAvgPrice + row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice)) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy new file mode 100644 index 000000000..fb9b88402 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy @@ -0,0 +1,158 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15Export { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["扬州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + CountDownLatch latch = new CountDownLatch(cities.size()) + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + executorService.execute(new Runnable() { + @Override + void run() { + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + +// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name) +// .append("weekend", new BasicDBObject('$lte', nextWeekDate)) +// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + }; + districtLatch.await() + row[6] = totalCount.get() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + }; + latch.await() + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + def row = [] + row[0] = city + row[6] = 0 + row[7] = 0 + result << row + } + } + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + executorService.shutdown() + districtExecutorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy new file mode 100644 index 000000000..40a866be1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy @@ -0,0 +1,130 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15ExportTest { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + + static main(args) { +// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", +// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", +// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", +// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", +// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", +// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", +// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", +// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def cities = ["常州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + final def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + int ppe15Count = 0 + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).each { obj -> + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + districtLatch.await() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + + } + }; + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + result << 0 + } + } + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy new file mode 100644 index 000000000..dbcf2139f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy @@ -0,0 +1,153 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 上海房屋租金导出 + * + * @author shenjian + * @since 2021/03/22 + */ +class RentExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static Map retMap = new ConcurrentHashMap() + + static main(args) { + def result = [] + result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"] + + int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).count() + AtomicInteger size = new AtomicInteger(count) + println("上海小区" + size.get()) + CountDownLatch latch = new CountDownLatch(count) + DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).each {obj -> + executorService.submit(new Runnable() { + @Override + void run() { + try { + DBObject query = new BasicDBObject() + query.append("city", "上海").append("district_name", obj.name).append("region", obj.region) + .append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22"))) + .append("house_type", new BasicDBObject('$lt', 5)) + + DBObject filter = new BasicDBObject() + filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1) + .append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1) + + buildResult(query, filter, result, obj.lng, obj.lat) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + println("*********************" + size.decrementAndGet()) + } + } + }) + } + + latch.await() + result.addAll(retMap.values()) + com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金"); + executorService.shutdown() + } + + static boolean buildResult (def query, def filter, def result, def lng, def lat) { + collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj -> + def row = [] + row[0] = obj.city + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.house_type + row[4] = obj.avg_price + row[5] = obj.sample_enum + row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d) + row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d) + + if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) { + // 判断价格是否合理 + if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) { + retMap.put(obj.district_name + obj.region + obj.house_type, row) + println(retMap.size()) + } else { + println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type) + } + } + } + } + + static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) { + if (retMap.contains(district_name + region + "1")) { + printf("一居室" + retMap.get(district_name + region + "1")[4]) + } + if (retMap.contains(district_name + region + "2")) { + printf("二居室" + retMap.get(district_name + region + "2")[4]) + } + if (retMap.contains(district_name + region + "3")) { + printf("三居室" + retMap.get(district_name + region + "3")[4]) + } + println("当前居室" + house_type + ", 价格为" + avg_price) + switch (house_type) { + case 2: + println(retMap.get(district_name + region + "1")) + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 3: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 4: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "3")) { + double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4]) + if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + } + return true + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy new file mode 100644 index 000000000..b0e7a120e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy @@ -0,0 +1,92 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class TotalTestExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["贵阳"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, 1) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, 1) + } + } + } + } + + + // 测试数据量(小区近一年成交量) + int tradeCount = 0 + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + if (map.get(obj.region + obj.district_name)) { + tradeCount++ + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } + row[6] = tradeCount + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + //int ppe15 = + + //row[7] = + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy new file mode 100644 index 000000000..82f000f52 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class ValueRateCityInfoExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["重庆","郑州", "长沙", "北京"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区数量 + int seaweedCount = DBUtils.seaweed.count( new BasicDBObject("city", city).append("cat", "district").append("visible", 0)) + + // 可估值小区数量 + int valuateCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 有样本估值数量及占比 + int haveSampleCount = collection.find(new BasicDBObject("city", city) + .append("value_type", 2) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 无样本估值数量及占比 + int notHaveSampleCount = valuateCount - haveSampleCount + + // 可信度大于0.5的数量 + int creditCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + .append("credit", new BasicDBObject('$gte', 0.5d)) + ).count() + + row[1] = seaweedCount + row[2] = valuateCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(valuateCount * 100) / Double.valueOf(seaweedCount)) + "%)" + if (valuateCount != 0) { + row[3] = haveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(haveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[4] = notHaveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(notHaveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[5] = creditCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(creditCount * 100) / Double.valueOf(valuateCount)) + "%)" + } + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.0-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy new file mode 100644 index 000000000..399b33c38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import org.apache.commons.lang3.time.DateUtils +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.* +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder + +import java.util.concurrent.atomic.AtomicLong + +/** + * 拷贝1.31前四周数据至3.28日缺失的四周数据中 + * 注意时间更新 + * + * @author shenjian + * @since 2021/3/30 + */ +class CopyDataForAnnPricePre { + + static RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + + static final long timestamp = (DateUtils.parseDate("2021-03-28", "yyyy-MM-dd").getTime() - + DateUtils.parseDate("2021-01-31", "yyyy-MM-dd").getTime()) + static final int days = timestamp / 1000 / 3600 / 24 + + static void main(String[] args) { + // 验证数据是否正确写入 + //String[] weeksNew= ["2021-03-07", "2021-03-14", "2021-03-21", "2021-03-28"] + String[] weeksNew= ["2021-03-07", "2021-01-31"] + for (String week : weeksNew) { + println(DateUtils.parseDate(week, "yyyy-MM-dd").getTime()) + } + //String[] weeksNew=["2021-03-28"] +// ///////////////////// 删除数据临时逻辑 +// // 先删除数据 +// DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("ann_price_pre") +// deleteRequest.setQuery(QueryBuilders.termQuery("date", DateUtils.parseDate("2021-03-07", "yyyy-MM-dd").getTime())) +// // 批次大小 +// deleteRequest.setBatchSize(1000); +// restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) +// ///////////////////// + checkNum(weeksNew) + + BulkRequest request = new BulkRequest(); + for (String week : weeksNew) { + AtomicLong count = new AtomicLong(0) + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.addDays(DateUtils.parseDate(week, "yyyy-MM-dd"), -days).getTime())) + .size(10000); + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + .source(searchSourceBuilder) + .scroll(scroll); // 设置scroll间隔 + + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //通过发送初始搜索请求来初始化搜索上下文 + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits() + write(count, searchHits, timestamp, request); + + while (searchHits != null && searchHits.length > 0) { //通过循环调用搜索滚动api来检索所有搜索命中,直到没有文档返回 + //处理返回的搜索结果 + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); //创建一个新的搜索滚动请求,保存最后返回的滚动标识符和滚动间隔 + scrollRequest.scroll(scroll); + searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + write(count, searchHits, timestamp, request); + } + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT) + println(week + "剩余批量操作成功:" + bulkResponse.status()) + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); //完成滚动后,清除滚动上下文 + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + boolean succeeded = clearScrollResponse.isSucceeded(); + println("清除滚动句柄" + succeeded) + } + checkNum(weeksNew) + + restHighLevelClient.close() + } + + static write(def count, def searchHits, def timestamp, def request) { + for (SearchHit hit : searchHits) { + Map map = hit.getSourceAsMap(); + map.put("date", Long.valueOf(map.get("date").toString()) + timestamp) + + request.add(new IndexRequest("ann_price_pre").source(map).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + if (count.get() >= 5000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + println("批量操作成功:" + bulkResponse.status()) + request = new BulkRequest(); + count = new AtomicLong(0) + } + } + } + + static checkNum(def weeksNew) { + // 验证数据是否正确写入 + for (String week : weeksNew) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.parseDate(week, "yyyy-MM-dd").getTime())); + + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + searchRequest.source(searchSourceBuilder) + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + + println(searchResponse.status().name() + " " + searchResponse.getHits().totalHits) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy new file mode 100644 index 000000000..68d16920d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy @@ -0,0 +1,98 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11,先写入mongo + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + QueryBuilder queryBuilder = QueryBuilders.matchAllQuery() + def districtPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) + //def newDistrictPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) +// RestHighLevelClient restHighLevelClient206 = getClient206() +// RestHighLevelClient restHighLevelClient = getClient() +// //def districtPriceStats +// GetResponse getResponse = getDocument(restHighLevelClient) + for (def map : districtPriceStats) { + DBObject dbObject = new BasicDBObject() + + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + fangjiaConn.save(dbObject) + +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.id(id).timeout(TimeValue.timeValueSeconds(1)) +// request.source(map) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); +// +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + //restHighLevelClient.close() + } + +// static RestHighLevelClient getClient() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.216",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static RestHighLevelClient getClient206() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.206",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static GetResponse getDocument(RestHighLevelClient restHighLevelClient206) throws IOException { +// SearchRequest searchRequest = new SearchRequest("district_price_stat"); +// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); +// //searchSourceBuilder.query(QueryBuilders.termQuery("url", "test")); +// searchRequest.source(searchSourceBuilder); +// SearchResponse response = restHighLevelClient206.search(searchRequest, RequestOptions.DEFAULT); +// System.out.println(response.getSourceAsString()); //获取文档的内容 +// return response +// } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy new file mode 100644 index 000000000..999f1eaf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import com.mongodb.DBCollection +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest + +import java.util.concurrent.atomic.AtomicLong + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11 + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat2 { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + // 先删除数据 + DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("district_price_stat") + deleteRequest.setQuery(QueryBuilders.matchAllQuery()) + // 批次大小 + deleteRequest.setBatchSize(1000); + restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) + + BulkRequest request = new BulkRequest(); + AtomicLong count = new AtomicLong() + fangjiaConn.find().batchSize(10000).each {map -> + Map dbObject = new HashMap() + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + request.add(new IndexRequest("district_price_stat").source(dbObject).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + // 每一千写入 + if (count.get() >= 1000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + request = new BulkRequest(); + count = new AtomicLong(0) + } + + // 单个插入文档 +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.timeout(TimeValue.timeValueSeconds(1)) +// request.source(dbObject) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); + +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + // 不足一千写入 + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + } + restHighLevelClient.close() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy new file mode 100644 index 000000000..bbbe4ebf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.temp.shenjian.es.util + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient + +/** + * @author shenjian + * @since 2021/3/30 + */ +class RestHighLevelClientUtils { + + static final def HOST = "192.168.0.216" + + static RestHighLevelClient getClient() { + RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(HOST,9200,"http")) + .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + httpClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpClientBuilder; + } + })) + return restHighLevelClient + } +}