From ddf08d464dabc5c81dea15ca20e5907307a58aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Apr 2021 12:34:04 +0800 Subject: [PATCH 001/118] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=BA=A7=E8=81=94?= =?UTF-8?q?=E5=9F=8E=E5=B8=82=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/DealPriceDetailExport.groovy | 183 ++++++++++++++++++ .../script/temp/shenjian/PPE15Export.groovy | 158 +++++++++++++++ .../temp/shenjian/PPE15ExportTest.groovy | 130 +++++++++++++ .../script/temp/shenjian/RentExport.groovy | 153 +++++++++++++++ .../temp/shenjian/TotalTestExport.groovy | 92 +++++++++ .../shenjian/ValueRateCityInfoExport.groovy | 91 +++++++++ .../shenjian/es/CopyDataForAnnPricePre.groovy | 124 ++++++++++++ .../shenjian/es/SyncDistrictPriceStat.groovy | 98 ++++++++++ .../shenjian/es/SyncDistrictPriceStat2.groovy | 78 ++++++++ .../es/util/RestHighLevelClientUtils.groovy | 35 ++++ 10 files changed, 1142 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy new file mode 100644 index 000000000..9f532247a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy @@ -0,0 +1,183 @@ +package com.fangjia.script.temp.shenjian + +import com.alibaba.excel.EasyExcel +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject + +import java.util.concurrent.* + +/** + * 成交详情导出 + * + * @author shenjian* @since 2021/3/29 + */ +class DealPriceDetailExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017) + static final Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(10) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州", + "合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台", + "徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁", + "南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华", + "昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城", + "芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州", + "连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口", + "西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"] +// def cities = ["常州"] + + def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + + CountDownLatch latch = new CountDownLatch(cities.size()) + DBObject filter = new BasicDBObject() + filter.put("city", 1) + filter.put("region", 1) + filter.put("district_name", 1) + filter.put("total_price", 1) + filter.put("trade_date", 1) + filter.put("avg_price", 1) + filter.put("area", 1) + filter.put("direction", 1) + filter.put("fitment", 1) + filter.put("height", 1) + filter.put("floor", 1) + filter.put("unit_num", 1) + filter.put("house_num", 1) + filter.put("room_num", 1) + filter.put("hall", 1) + filter.put("room", 1) + filter.put("toilet", 1) + for (def city : cities) { + final currCity = city + executorService.execute(new Runnable() { + @Override + void run() { + try { + final def ret = new CopyOnWriteArrayList() + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + //ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"] + DBObject query = new BasicDBObject(); + query.put("city", currCity) + query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))) + + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(query, filter).batchSize(10000).each { tempObj -> + final obj = tempObj + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def row = [] + row[0] = currCity + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.total_price + row[4] = obj.avg_price + if (obj.trade_date) { + row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd") + } + row[6] = obj.area + row[7] = obj.direction + row[8] = obj.fitment + row[9] = obj.height + row[10] = obj.floor + row[11] = obj.unit_num + row[12] = obj.house_num + row[13] = obj.room_num + row[14] = obj.hall + row[15] = obj.room + row[16] = obj.toilet + def objOur = map.get(obj.region + obj.district_name) + calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row) + ret.add(row) + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + println("出现异常111111.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + } + districtLatch.await() + EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret); + //ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息") + println("当前处理城市" + currCity + " " + ret.size()) + } catch (Exception ex) { + println("出现异常22222.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + executorService.shutdown() + districtExecutorService.shutdown() + } + + private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) { + // 最相近一周的该小区均价 + Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", objOur.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name) + row[17] = "" + row[18] = "" + } else { + row[17] = weekendAvgPrice + row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice)) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy new file mode 100644 index 000000000..fb9b88402 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy @@ -0,0 +1,158 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15Export { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["扬州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + CountDownLatch latch = new CountDownLatch(cities.size()) + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + executorService.execute(new Runnable() { + @Override + void run() { + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + +// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name) +// .append("weekend", new BasicDBObject('$lte', nextWeekDate)) +// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + }; + districtLatch.await() + row[6] = totalCount.get() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + }; + latch.await() + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + def row = [] + row[0] = city + row[6] = 0 + row[7] = 0 + result << row + } + } + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + executorService.shutdown() + districtExecutorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy new file mode 100644 index 000000000..40a866be1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy @@ -0,0 +1,130 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15ExportTest { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + + static main(args) { +// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", +// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", +// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", +// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", +// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", +// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", +// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", +// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def cities = ["常州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + final def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + int ppe15Count = 0 + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).each { obj -> + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + districtLatch.await() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + + } + }; + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + result << 0 + } + } + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy new file mode 100644 index 000000000..dbcf2139f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy @@ -0,0 +1,153 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 上海房屋租金导出 + * + * @author shenjian + * @since 2021/03/22 + */ +class RentExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static Map retMap = new ConcurrentHashMap() + + static main(args) { + def result = [] + result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"] + + int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).count() + AtomicInteger size = new AtomicInteger(count) + println("上海小区" + size.get()) + CountDownLatch latch = new CountDownLatch(count) + DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).each {obj -> + executorService.submit(new Runnable() { + @Override + void run() { + try { + DBObject query = new BasicDBObject() + query.append("city", "上海").append("district_name", obj.name).append("region", obj.region) + .append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22"))) + .append("house_type", new BasicDBObject('$lt', 5)) + + DBObject filter = new BasicDBObject() + filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1) + .append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1) + + buildResult(query, filter, result, obj.lng, obj.lat) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + println("*********************" + size.decrementAndGet()) + } + } + }) + } + + latch.await() + result.addAll(retMap.values()) + com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金"); + executorService.shutdown() + } + + static boolean buildResult (def query, def filter, def result, def lng, def lat) { + collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj -> + def row = [] + row[0] = obj.city + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.house_type + row[4] = obj.avg_price + row[5] = obj.sample_enum + row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d) + row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d) + + if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) { + // 判断价格是否合理 + if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) { + retMap.put(obj.district_name + obj.region + obj.house_type, row) + println(retMap.size()) + } else { + println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type) + } + } + } + } + + static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) { + if (retMap.contains(district_name + region + "1")) { + printf("一居室" + retMap.get(district_name + region + "1")[4]) + } + if (retMap.contains(district_name + region + "2")) { + printf("二居室" + retMap.get(district_name + region + "2")[4]) + } + if (retMap.contains(district_name + region + "3")) { + printf("三居室" + retMap.get(district_name + region + "3")[4]) + } + println("当前居室" + house_type + ", 价格为" + avg_price) + switch (house_type) { + case 2: + println(retMap.get(district_name + region + "1")) + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 3: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 4: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "3")) { + double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4]) + if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + } + return true + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy new file mode 100644 index 000000000..b0e7a120e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy @@ -0,0 +1,92 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class TotalTestExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["贵阳"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, 1) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, 1) + } + } + } + } + + + // 测试数据量(小区近一年成交量) + int tradeCount = 0 + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + if (map.get(obj.region + obj.district_name)) { + tradeCount++ + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } + row[6] = tradeCount + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + //int ppe15 = + + //row[7] = + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy new file mode 100644 index 000000000..82f000f52 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class ValueRateCityInfoExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["重庆","郑州", "长沙", "北京"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区数量 + int seaweedCount = DBUtils.seaweed.count( new BasicDBObject("city", city).append("cat", "district").append("visible", 0)) + + // 可估值小区数量 + int valuateCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 有样本估值数量及占比 + int haveSampleCount = collection.find(new BasicDBObject("city", city) + .append("value_type", 2) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 无样本估值数量及占比 + int notHaveSampleCount = valuateCount - haveSampleCount + + // 可信度大于0.5的数量 + int creditCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + .append("credit", new BasicDBObject('$gte', 0.5d)) + ).count() + + row[1] = seaweedCount + row[2] = valuateCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(valuateCount * 100) / Double.valueOf(seaweedCount)) + "%)" + if (valuateCount != 0) { + row[3] = haveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(haveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[4] = notHaveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(notHaveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[5] = creditCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(creditCount * 100) / Double.valueOf(valuateCount)) + "%)" + } + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.0-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy new file mode 100644 index 000000000..399b33c38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import org.apache.commons.lang3.time.DateUtils +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.* +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder + +import java.util.concurrent.atomic.AtomicLong + +/** + * 拷贝1.31前四周数据至3.28日缺失的四周数据中 + * 注意时间更新 + * + * @author shenjian + * @since 2021/3/30 + */ +class CopyDataForAnnPricePre { + + static RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + + static final long timestamp = (DateUtils.parseDate("2021-03-28", "yyyy-MM-dd").getTime() - + DateUtils.parseDate("2021-01-31", "yyyy-MM-dd").getTime()) + static final int days = timestamp / 1000 / 3600 / 24 + + static void main(String[] args) { + // 验证数据是否正确写入 + //String[] weeksNew= ["2021-03-07", "2021-03-14", "2021-03-21", "2021-03-28"] + String[] weeksNew= ["2021-03-07", "2021-01-31"] + for (String week : weeksNew) { + println(DateUtils.parseDate(week, "yyyy-MM-dd").getTime()) + } + //String[] weeksNew=["2021-03-28"] +// ///////////////////// 删除数据临时逻辑 +// // 先删除数据 +// DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("ann_price_pre") +// deleteRequest.setQuery(QueryBuilders.termQuery("date", DateUtils.parseDate("2021-03-07", "yyyy-MM-dd").getTime())) +// // 批次大小 +// deleteRequest.setBatchSize(1000); +// restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) +// ///////////////////// + checkNum(weeksNew) + + BulkRequest request = new BulkRequest(); + for (String week : weeksNew) { + AtomicLong count = new AtomicLong(0) + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.addDays(DateUtils.parseDate(week, "yyyy-MM-dd"), -days).getTime())) + .size(10000); + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + .source(searchSourceBuilder) + .scroll(scroll); // 设置scroll间隔 + + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //通过发送初始搜索请求来初始化搜索上下文 + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits() + write(count, searchHits, timestamp, request); + + while (searchHits != null && searchHits.length > 0) { //通过循环调用搜索滚动api来检索所有搜索命中,直到没有文档返回 + //处理返回的搜索结果 + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); //创建一个新的搜索滚动请求,保存最后返回的滚动标识符和滚动间隔 + scrollRequest.scroll(scroll); + searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + write(count, searchHits, timestamp, request); + } + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT) + println(week + "剩余批量操作成功:" + bulkResponse.status()) + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); //完成滚动后,清除滚动上下文 + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + boolean succeeded = clearScrollResponse.isSucceeded(); + println("清除滚动句柄" + succeeded) + } + checkNum(weeksNew) + + restHighLevelClient.close() + } + + static write(def count, def searchHits, def timestamp, def request) { + for (SearchHit hit : searchHits) { + Map map = hit.getSourceAsMap(); + map.put("date", Long.valueOf(map.get("date").toString()) + timestamp) + + request.add(new IndexRequest("ann_price_pre").source(map).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + if (count.get() >= 5000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + println("批量操作成功:" + bulkResponse.status()) + request = new BulkRequest(); + count = new AtomicLong(0) + } + } + } + + static checkNum(def weeksNew) { + // 验证数据是否正确写入 + for (String week : weeksNew) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.parseDate(week, "yyyy-MM-dd").getTime())); + + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + searchRequest.source(searchSourceBuilder) + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + + println(searchResponse.status().name() + " " + searchResponse.getHits().totalHits) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy new file mode 100644 index 000000000..68d16920d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy @@ -0,0 +1,98 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11,先写入mongo + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + QueryBuilder queryBuilder = QueryBuilders.matchAllQuery() + def districtPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) + //def newDistrictPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) +// RestHighLevelClient restHighLevelClient206 = getClient206() +// RestHighLevelClient restHighLevelClient = getClient() +// //def districtPriceStats +// GetResponse getResponse = getDocument(restHighLevelClient) + for (def map : districtPriceStats) { + DBObject dbObject = new BasicDBObject() + + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + fangjiaConn.save(dbObject) + +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.id(id).timeout(TimeValue.timeValueSeconds(1)) +// request.source(map) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); +// +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + //restHighLevelClient.close() + } + +// static RestHighLevelClient getClient() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.216",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static RestHighLevelClient getClient206() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.206",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static GetResponse getDocument(RestHighLevelClient restHighLevelClient206) throws IOException { +// SearchRequest searchRequest = new SearchRequest("district_price_stat"); +// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); +// //searchSourceBuilder.query(QueryBuilders.termQuery("url", "test")); +// searchRequest.source(searchSourceBuilder); +// SearchResponse response = restHighLevelClient206.search(searchRequest, RequestOptions.DEFAULT); +// System.out.println(response.getSourceAsString()); //获取文档的内容 +// return response +// } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy new file mode 100644 index 000000000..999f1eaf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import com.mongodb.DBCollection +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest + +import java.util.concurrent.atomic.AtomicLong + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11 + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat2 { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + // 先删除数据 + DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("district_price_stat") + deleteRequest.setQuery(QueryBuilders.matchAllQuery()) + // 批次大小 + deleteRequest.setBatchSize(1000); + restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) + + BulkRequest request = new BulkRequest(); + AtomicLong count = new AtomicLong() + fangjiaConn.find().batchSize(10000).each {map -> + Map dbObject = new HashMap() + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + request.add(new IndexRequest("district_price_stat").source(dbObject).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + // 每一千写入 + if (count.get() >= 1000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + request = new BulkRequest(); + count = new AtomicLong(0) + } + + // 单个插入文档 +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.timeout(TimeValue.timeValueSeconds(1)) +// request.source(dbObject) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); + +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + // 不足一千写入 + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + } + restHighLevelClient.close() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy new file mode 100644 index 000000000..bbbe4ebf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.temp.shenjian.es.util + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient + +/** + * @author shenjian + * @since 2021/3/30 + */ +class RestHighLevelClientUtils { + + static final def HOST = "192.168.0.216" + + static RestHighLevelClient getClient() { + RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(HOST,9200,"http")) + .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + httpClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpClientBuilder; + } + })) + return restHighLevelClient + } +} From d6f7803c6cd27b7263a2b3e539b714289afceef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Apr 2021 14:38:53 +0800 Subject: [PATCH 002/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=95=86=E5=9C=88?= =?UTF-8?q?=E5=9C=B0=E5=9D=80=E7=BB=8F=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/BusinessDianpingAddr.groovy | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy new file mode 100644 index 000000000..dd8719a3f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy @@ -0,0 +1,60 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +/** + * 获取商圈地标 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingAddr { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + + static void main(String[] args) { + def cities = ["苏州", "西安", "合肥"] + for (def city : cities) { + DBObject query = new BasicDBObject() + query.put("city", city) + collection.find(query).batchSize(300).each {obj -> + def id = obj._id + if (obj.lng == null || obj.lat == null) { + // 查询坐标位置,并更新 + def location = this.parserByGaode(city, obj.address) + if (location != null) { + DBObject updateField = new BasicDBObject() + updateField.put("lng", location.lng) + updateField.put("lat", location.lat) + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', updateField), false, true) + } + } + } + } + collection.find() + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + if (str && ["门牌号", "单元号", "热点商圈", "兴趣点"].contains(level)) { + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } else { + println address + " " + level + return null + } + } + } + } +} From a4717f4e83b7f0a8a1d015f119278bbc6185094b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Apr 2021 19:50:58 +0800 Subject: [PATCH 003/118] =?UTF-8?q?=E5=95=86=E5=9C=88=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BusinessDianpingAddr.groovy | 4 +- .../business/BusinessDianpingExport.groovy | 95 +++++++++++++++++++ .../business/BusinessDianpingModify.groovy | 29 ++++++ 3 files changed, 125 insertions(+), 3 deletions(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{ => business}/BusinessDianpingAddr.groovy (95%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy similarity index 95% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy index dd8719a3f..934c3411d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.temp.shenjian +package com.fangjia.script.temp.shenjian.business import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -6,7 +6,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import net.sf.json.JSONObject -import org.bson.types.ObjectId /** * 获取商圈地标 @@ -37,7 +36,6 @@ class BusinessDianpingAddr { } } } - collection.find() } static def parserByGaode(def city, def address) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy new file mode 100644 index 000000000..b18a14517 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -0,0 +1,95 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈地标导出 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def cities = ["苏州", "西安", "合肥"] + def headers = ["城市", "区域", "商圈名称", "地标名称(3-5个)", "地标地址(与地标名称对应)", "地标经度(与地标名称对应)", "地标纬度(与地标名称对应)", "备注", "创建时间", "更新时间", "来源公司"]; + for (int i = 0; i < cities.size(); i++) { + final def city = cities[i]; + def result = [] + // 获取所有商圈信息 + def businessNames = collection.distinct("name", new BasicDBObject("city", city)) + // 获取商圈下3个地标 + for (def name : businessNames) { + List landmarks = new ArrayList<>() + List addresses = new ArrayList<>() + List lngs = new ArrayList<>() + List lats = new ArrayList<>() + String region = "" + String c_date + String m_date + int landmarkLen = 0; + collection.find(new BasicDBObject("city", city).append("name", name)).sort(new BasicDBObject("order", 1)).each { obj -> + // 按人气取经纬度不为空的地标 + if (landmarks.size() < 5) { + // 特殊标题去重 + if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1 + || String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1 + || String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1 + || String.valueOf(obj.landmark).indexOf(",") != -1) { + + } else { + // 防止导入导致字段过长,一个字符2个字节 + if (landmarkLen + String.valueOf(obj.landmark).length() <= 100) { + landmarkLen += String.valueOf(obj.landmark).length() + landmarks.add(obj.landmark) + addresses.add(obj.address) + lngs.add(obj.lng) + lats.add(obj.lat) + + if (StringUtils.isBlank(region)) { + region = obj.region + c_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss"); + m_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.m_date, "yyyy-MM-dd HH:mm:ss"); + } + } + } + } + } + def ret = [] + def city_code + def district_code + if (StringUtils.isNotBlank(region)) { + cityCollection.find(new BasicDBObject("fangjia_city", city).append("district", new BasicDBObject('$regex', region + "+")), + new BasicDBObject("city_code", 1).append("district_code", 1)).each {obj -> + city_code = obj.city_code + district_code = obj.district_code + } + } + ret[0] = city_code + ret[1] = district_code + ret[2] = name + ret[3] = StringUtils.join(landmarks, ",") + ret[4] = StringUtils.join(addresses, ",") + ret[5] = StringUtils.join(lngs, ",") + ret[6] = StringUtils.join(lats, ",") + ret[7] = "" + ret[8] = c_date + ret[9] = m_date + ret[10] = "fjw" + if (district_code != null) { + result << ret + } + } + FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy new file mode 100644 index 000000000..dd0c4c2dc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈数据修正 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingModify { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def city = "合肥" + def origin = "政务区" + def now = "蜀山区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + } +} From 77aa3327e296a540f655d935264c7f6bac0d77c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Apr 2021 14:47:34 +0800 Subject: [PATCH 004/118] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E5=AE=8C=E5=96=84=E4=B8=8E=E7=A9=BA=E6=A0=BC=E5=A4=84?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/business/BusinessDianpingAddr.groovy | 11 +++-------- .../shenjian/business/BusinessDianpingExport.groovy | 7 ++++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy index 934c3411d..9e3d9b54e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy @@ -26,7 +26,7 @@ class BusinessDianpingAddr { def id = obj._id if (obj.lng == null || obj.lat == null) { // 查询坐标位置,并更新 - def location = this.parserByGaode(city, obj.address) + def location = this.parserByGaode(city, obj.region + obj.name + obj.address) if (location != null) { DBObject updateField = new BasicDBObject() updateField.put("lng", location.lng) @@ -45,13 +45,8 @@ class BusinessDianpingAddr { if (ret) { def str = ret.get(0).location def level = ret.get(0).level - if (str && ["门牌号", "单元号", "热点商圈", "兴趣点"].contains(level)) { - def arrs = str.split(",") - return ["lng": arrs[0], "lat": arrs[1]] - } else { - println address + " " + level - return null - } + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy index b18a14517..dca87e75e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -48,9 +48,10 @@ class BusinessDianpingExport { } else { // 防止导入导致字段过长,一个字符2个字节 - if (landmarkLen + String.valueOf(obj.landmark).length() <= 100) { - landmarkLen += String.valueOf(obj.landmark).length() - landmarks.add(obj.landmark) + def landmark = String.valueOf(obj.landmark).replaceAll(" ", "") + if (landmarkLen + String.valueOf(landmark).length() <= 100) { + landmarkLen += String.valueOf(landmark).length() + landmarks.add(landmark) addresses.add(obj.address) lngs.add(obj.lng) lats.add(obj.lat) From 5b75537604009c717ee81a5adb2002f932e5dca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Apr 2021 18:42:09 +0800 Subject: [PATCH 005/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=95=86=E5=9C=88?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/BusinessDianpingExport.groovy | 6 +++-- .../business/BusinessDianpingModify.groovy | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy index dca87e75e..3f03c1c01 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -44,12 +44,12 @@ class BusinessDianpingExport { if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1 || String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1 || String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1 - || String.valueOf(obj.landmark).indexOf(",") != -1) { + || String.valueOf(obj.landmark).indexOf(",") != -1 || String.valueOf(obj.landmark).indexOf("商圈附近") != -1 || String.valueOf(obj.landmark).indexOf("】") != -1) { } else { // 防止导入导致字段过长,一个字符2个字节 def landmark = String.valueOf(obj.landmark).replaceAll(" ", "") - if (landmarkLen + String.valueOf(landmark).length() <= 100) { + if (landmark.length() <= 30 && landmarkLen + String.valueOf(landmark).length() <= 100) { landmarkLen += String.valueOf(landmark).length() landmarks.add(landmark) addresses.add(obj.address) @@ -88,6 +88,8 @@ class BusinessDianpingExport { ret[10] = "fjw" if (district_code != null) { result << ret + } else { + println("未找到商圈地标:" + name) } } FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy index dd0c4c2dc..df968e183 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy @@ -25,5 +25,27 @@ class BusinessDianpingModify { collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) } + + city = "苏州" + origin = "工业园区" + now = "吴中区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + city = "苏州" + origin = "甪直" + now = "吴中区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + city = "西安" + origin = "高新区" + now = "雁塔区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + } } From 01e5dff436d96798c84af593db093fe65d8dfd23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 15 Apr 2021 14:16:36 +0800 Subject: [PATCH 006/118] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=88=AB=E5=90=8D?= =?UTF-8?q?=E8=A6=86=E7=9B=96=E7=8E=87=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../seaweed/SeaweedAliasExport.groovy | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy new file mode 100644 index 000000000..f8c9beb67 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy @@ -0,0 +1,63 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 小区别名数量统计 + * + * @author shenjian + * @since 2021/4/15 + */ +class SeaweedAliasExport { + + static void main(String[] args) { + def cities = ["重庆","苏州","昆明","柳州","曲靖","三明","达州","石狮","大理","福安","平潭","迪庆州"] + def result = [] + result << ["城市", "小区数量", "别名数量", "别名覆盖率"] + for (def city : cities) { + def ret = [] + + DBObject query = new BasicDBObject(); + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) + + //def seaweedCount = DBUtils.seaweed.find(query).batchSize(5000).count() + + int seaweedCount = 0; + int aliasCount = 0; + + if (city == "平潭") { + query.put("city", "福州") + query.put("region", city) + } + if (city == "福安") { + query.put("city", "宁德") + query.put("region", city) + } + if (city == "石狮") { + query.put("city", "泉州") + query.put("region", city) + } + DBUtils.seaweed.find(query).batchSize(5000).each {obj -> + if (obj.alias) { + aliasCount++ + } + seaweedCount++ + } + + ret[0] = city + ret[1] = seaweedCount + ret[2] = aliasCount + if (seaweedCount > 0) { + ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(aliasCount) * 100 / Double.valueOf(seaweedCount)) + } + result << ret + } + println(result) + ExcelUtils.write("F:\\数据\\小区别名覆盖率.xlsx", result, "统计") + } +} From 4b07cf082fab3ccd68c2f47fe2e1bfdba326e4ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 15 Apr 2021 14:21:23 +0800 Subject: [PATCH 007/118] =?UTF-8?q?easyexcel=E5=BC=95=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index bb7959032..1a14ef3ee 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -476,6 +476,11 @@ aliyun-java-sdk-address-purification 1.0.0 + + com.alibaba + easyexcel + 2.2.8 + From d76d663419b5a44f709848d4dba903e41c5358ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 16 Apr 2021 17:22:07 +0800 Subject: [PATCH 008/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/school/DistanceDeal.groovy | 86 +++++++++++++++++++ .../shenjian/school/FillMidSchoolInfo.groovy | 59 +++++++++++++ .../FillMidSchoolWithPrimarySchool.groovy | 56 ++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy new file mode 100644 index 000000000..001bfd9b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.core.util.AddressLatlng +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +/** + * 处理小区-学校距离过大的数据 + * + * @author shenjian + * @since 2021/4/16 + */ +class DistanceDeal { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) + + // 修复 鑫苑国际城市花园 https://guojichengshihuayuanszxy.fang.com 区域问题 + static void main(String[] args) { +// def result = GaoDeApiUtils.geocode("120.701271,31.32798") +// print(result) + + /* def ret = AddressLatlng.getLngLatByAddress("苏州", "苏州工业园区星湾学校(西校区)") + print(ret)*/ + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", "120.68764") +// updateField.put("school_lat", "31.330963") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true) + + DBObject updateField = new BasicDBObject() + updateField.put("school_lng", 120.68764) + updateField.put("school_lat", 31.330963) + updateField.put("school_name", "星湾学校西校区") + updateField.put("school_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") + updateField.put("gaode_name", "星湾学校西校区") + updateField.put("gaode_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") + + collection.update(new BasicDBObject("match_name", new BasicDBObject('$in', ["星海人家", "新加花园", "中央景城", "加城湖滨公寓", "沁苑小区", "湖畔花园", + "中茵皇冠国际", "天域三期", "天域", "天域二期"])).append("match_region", "吴中"), new BasicDBObject('$set', updateField), false, true) + +// def location = this.parserByGaode("苏州", "苏州工业园区星湾学校(西校区)") +// print(location) + +// if (location != null) { + //DBObject updateField = new BasicDBObject() +// updateField.put("match_lng", "120.614203") +// updateField.put("match_lat", "31.309876") +// collection.update(new BasicDBObject("url", "https://jingdelu0512.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "120.610651479") +// updateField.put("match_lat", "31.3131018333") +// collection.update(new BasicDBObject("url", "https://wuqufang.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "117.2703018188") +// updateField.put("match_lat", "31.7484989166") +// collection.update(new BasicDBObject("url", "https://binghuqiancheng.fang.com"), new BasicDBObject('$set', updateField), false, true) + +// +// updateField.put("match_lng", "120.610343") +// updateField.put("match_lat", "31.312715") +// collection.update(new BasicDBObject("url", "https://baolinsiqian.fang.com"), new BasicDBObject('$set', updateField), false, true) +// } +// DBObject updateField = new BasicDBObject() +// updateField.put("match_region", "蜀山") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10689.htm"), new BasicDBObject('$set', updateField), false, true) +// } + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy new file mode 100644 index 000000000..b621eb9fd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolInfo { + + static void main(String[] args) { + Map> map = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + String districtName = String.valueOf(data.get(2)) + if (districtName.contains("期")) { + String key = districtName.substring(0, districtName.length() - 2); + if (data.get(11) != null) { // 存在对口初中 + map.put(key, data); + } + } + } + print(map.size()) + Long count = 0; + for (List data : list) { + String districtName = String.valueOf(data.get(2)) + String key + if (districtName.contains("期")) { + key = districtName.substring(0, districtName.length() - 2) + } else { + key = districtName + } + if (map.get(key) != null && data.get(11) == null) { + List val = map.get(key) + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + count++ + println(data) + } + } + + print("优化数量" + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + list.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy new file mode 100644 index 000000000..b67c5a375 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolWithPrimarySchool { + + static void main(String[] args) { + Map> map = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + // 对口小学 对口中学 + String city = data.get(0) + String region = data.get(1) + if (data.get(7) != null && data.get(11) != null) { + map.put(city + region + data.get(7), data) + } + } + print(map.size()) + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String primarySchool = data.get(7) + + String key = city + region + primarySchool + if (map.get(key) != null && data.get(11) == null) { + List val = map.get(key) + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + count++ + println(data) + } + } + + print("优化数量" + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + list.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据") + } +} From 8cad854b3a337b489f660cb92040dc2ef43d24da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 16 Apr 2021 17:22:07 +0800 Subject: [PATCH 009/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/school/DistanceDeal.groovy | 90 ++++++++++++++++ .../shenjian/school/FillMidSchoolInfo.groovy | 101 ++++++++++++++++++ .../FillMidSchoolWithPrimarySchool.groovy | 57 ++++++++++ .../temp/shenjian/school/Standard.groovy | 67 ++++++++++++ 4 files changed, 315 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy new file mode 100644 index 000000000..d19524f07 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -0,0 +1,90 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.core.util.AddressLatlng +import com.fangjia.script.district.Distance +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +/** + * 处理小区-学校距离过大的数据 + * + * @author shenjian + * @since 2021/4/16 + */ +class DistanceDeal { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) + + // 修复 鑫苑国际城市花园 https://guojichengshihuayuanszxy.fang.com 区域问题 + static void main(String[] args) { + def result = GaoDeApiUtils.geocode("117.363998413086,31.89208984375") + print(result) + + /* def ret = AddressLatlng.getLngLatByAddress("苏州", "苏州工业园区星湾学校(西校区)") + print(ret)*/ + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", "120.68764") +// updateField.put("school_lat", "31.330963") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true) + +// double length = Distance.GetDistance(); +// print(length) + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", 120.68764) +// updateField.put("school_lat", 31.330963) +// updateField.put("school_name", "星湾学校西校区") +// updateField.put("school_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") +// updateField.put("gaode_name", "星湾学校西校区") +// updateField.put("gaode_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") +// +// collection.update(new BasicDBObject("match_name", new BasicDBObject('$in', ["星海人家", "新加花园", "中央景城", "加城湖滨公寓", "沁苑小区", "湖畔花园", +// "中茵皇冠国际", "天域三期", "天域", "天域二期"])).append("match_region", "吴中"), new BasicDBObject('$set', updateField), false, true) + +// def location = this.parserByGaode("苏州", "苏州工业园区星湾学校(西校区)") +// print(location) + +// if (location != null) { + //DBObject updateField = new BasicDBObject() +// updateField.put("match_lng", "120.614203") +// updateField.put("match_lat", "31.309876") +// collection.update(new BasicDBObject("url", "https://jingdelu0512.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "120.610651479") +// updateField.put("match_lat", "31.3131018333") +// collection.update(new BasicDBObject("url", "https://wuqufang.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "117.2703018188") +// updateField.put("match_lat", "31.7484989166") +// collection.update(new BasicDBObject("url", "https://binghuqiancheng.fang.com"), new BasicDBObject('$set', updateField), false, true) + +// +// updateField.put("match_lng", "120.610343") +// updateField.put("match_lat", "31.312715") +// collection.update(new BasicDBObject("url", "https://baolinsiqian.fang.com"), new BasicDBObject('$set', updateField), false, true) +// } +// DBObject updateField = new BasicDBObject() +// updateField.put("match_region", "蜀山") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10689.htm"), new BasicDBObject('$set', updateField), false, true) +// } + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy new file mode 100644 index 000000000..95c48476d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy @@ -0,0 +1,101 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang3.StringUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolInfo { + + static void main(String[] args) { + Map> map = new HashMap<>(); + Map> map2 = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String districtName = String.valueOf(data.get(2)) + String key = city + region + if (districtName.contains("期")) { + key += districtName.substring(0, districtName.length() - 2); + if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中 + map.put(key, data); + } + if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学 + map2.put(key, data); + } + } else { + key += districtName + if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中 + map.put(key, data); + } + if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学 + map2.put(key, data); + } + } + } + print(map.size()) + Long countPrimary = 0; + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String districtName = String.valueOf(data.get(2)) + String key = city + region + if (districtName.contains("期")) { + key += districtName.substring(0, districtName.length() - 2) + } else { + key += districtName + } + if (map.get(key) != null) { + List val = map.get(key) + if (StringUtils.isBlank(data.get(11))) { + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + + count++ + println("初中" + data) + } + } + if (map2.get(key) != null) { + List val = map2.get(key) + if (StringUtils.isBlank(data.get(6))) { + data.set(6, val.get(6)) + data.set(7, val.get(7)) + data.set(8, val.get(8)) + data.set(9, val.get(9)) + data.set(10, val.get(10)) + + countPrimary++ + println("小学" + data) + } + } + } + + List> result = new ArrayList<>(); + for (List data : list) { + if (StringUtils.isNotBlank(data.get(6))) { + result.add(data) + } + } + + print("优化数量" + countPrimary + " " + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + result.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", result, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy new file mode 100644 index 000000000..8f4ace4c8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang3.StringUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolWithPrimarySchool { + + static void main(String[] args) { + Map> map = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + // 对口小学 对口中学 + String city = data.get(0) + String region = data.get(1) + if (org.apache.commons.lang.StringUtils.isNotBlank(data.get(7)) && StringUtils.isNotBlank(data.get(11))) { + map.put(city + region + data.get(7), data) + } + } + print(map.size()) + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String primarySchool = data.get(7) + + String key = city + region + primarySchool + if (map.get(key) != null && StringUtils.isBlank(data.get(11))) { + List val = map.get(key) + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + count++ + println(data) + } + } + + print("优化数量" + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + list.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy new file mode 100644 index 000000000..989f77bc6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.ExcelUtils + +/** + * 标准化文件输出 + * + * @author shenjian + * @since 2021/4/16 + */ +class Standard { + + static void main(String[] args) { + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V2.0.xlsx") + def resultOne = [] + def resultTwo = [] + def resultThree = [] + for (List data : list) { + // "320500","610100","340100" + if (data.get(0) == "320500") { + saveData(resultOne, data) + } else if (data.get(0) == "610000") { + data.set(0, 610100) + saveData(resultTwo, data) + } else if (data.get(0) == "340100") { + saveData(resultThree, data) + } + } + def headers = ["城市","区域","小区名称","小区经度","小区纬度", "小区地址","对口小学","小学地址","小学属性","小学批次","小学经度","小学纬度","对口中学","中学地址","中学属性", + "中学批次","中学经度","中学纬度","备注","创建时间","修改时间","来源网站"] + FileUtils.batchCreateFile(resultOne, "F:\\工作\\数据采购\\学校\\bigdata_commschool_320500.dat", headers, "utf-8") + FileUtils.batchCreateFile(resultTwo, "F:\\工作\\数据采购\\学校\\bigdata_commschool_610100.dat", headers, "utf-8") + FileUtils.batchCreateFile(resultThree, "F:\\工作\\数据采购\\学校\\bigdata_commschool_340100.dat", headers, "utf-8") + } + + static void saveData(def result, def data) { + def ret = [] + ret[0] = data.get(0) + ret[1] = data.get(1) + ret[2] = data.get(2) + ret[3] = data.get(3) != null && data.get(3) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(3))) : "" + + ret[4] = data.get(4) != null && data.get(4) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(4))) : "" + ret[5] = data.get(5) + + ret[6] = data.get(6) + ret[7] = data.get(7) + ret[8] = data.get(8) + ret[9] = "" + ret[10] = data.get(9) != null && data.get(9) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(9))) : "" + ret[11] = data.get(10) != null && data.get(10) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(10))) : "" + ret[12] = data.get(11) + ret[13] = data.get(12) + ret[14] = data.get(13) + ret[15] = "" + ret[16] = data.get(14) != null && data.get(14) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(14))) : "" + ret[17] = data.get(15) != null && data.get(15) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(15))) : "" + ret[18] = data.get(16) + ret[19] = data.get(17) + ret[20] = data.get(18) + ret[21] = data.get(19) + + result << ret + } +} From f1e7cc1d442c4a2278297cbc430542a3aaddd785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 21 Apr 2021 14:39:40 +0800 Subject: [PATCH 010/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=88=BF=E5=A4=A9?= =?UTF-8?q?=E4=B8=8B=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/CommFang.groovy | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy new file mode 100644 index 000000000..5da9281bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -0,0 +1,54 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 贝壳找房小区种子生成 + */ +class CommFang { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source","搜房") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.district_id + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf(".")) + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", "fang.com") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + set.add(id) + doc.add(dbObject) + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file From 040bbc5eca954da5d1b3c8726189b6f6f6214dab Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Wed, 21 Apr 2021 15:21:34 +0800 Subject: [PATCH 011/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E5=B0=8F=E5=8C=BA=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/CommFang.groovy | 2 +- .../script/temp/shenjian/crawl/CommKe.groovy | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy index 5da9281bd..a36b31539 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -6,7 +6,7 @@ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.time.DateUtils /** - * 贝壳找房小区种子生成 + * 房天下小区种子生成 */ class CommFang { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy new file mode 100644 index 000000000..0e2baf552 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy @@ -0,0 +1,57 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 贝壳找房小区种子生成 + */ +class CommKe { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static final def source = "贝壳找房" + static final def root = "ke.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("url", 1) + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.url + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = obj.district_id + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + set.add(id) + doc.add(dbObject) + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file From a2081cf9df8600e73b8bc474b330f029ff6f6a8e Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Tue, 27 Apr 2021 09:32:35 +0800 Subject: [PATCH 012/118] =?UTF-8?q?=E6=97=A0=E6=95=88=E7=A7=8D=E5=AD=90?= =?UTF-8?q?=E6=B8=85=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/crawl/CommFang.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy index a36b31539..e44632fc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -39,8 +39,10 @@ class CommFang { dbObject.put("c_date", c_date) dbObject.put("m_date", c_date) - set.add(id) - doc.add(dbObject) + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } if (doc.size() == 5000) { districtSeeds.insert(doc) doc.clear() From 7d05160e818177be19bfcbda28db1ec5abe9ed94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 30 Apr 2021 10:38:34 +0800 Subject: [PATCH 013/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E7=A6=8F=E5=B7=9E?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/crawl/Comm58.groovy | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy new file mode 100644 index 000000000..3aa674810 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy @@ -0,0 +1,68 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.time.DateUtils + +/** + * @author shenjian + * @since 2021/4/30 + */ +class Comm58 { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static final def source = "58同城" + static final def root = "58.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("city","福州") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(10000).each { obj -> + try { + def url = obj.district_id + def cityShort = "fz" + + String[] info = url.split("/") + def id = info[info.size() - 1] + if (!set.contains(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("type", "comm") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } catch(Exception ex) { + println("解析异常:" + obj) + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } + +} From 4c3d27236c8e8dc88dbe98d093d7d278cd0d6f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 30 Apr 2021 10:41:27 +0800 Subject: [PATCH 014/118] =?UTF-8?q?=E7=A7=8D=E5=AD=90=E9=9B=86=E5=90=88?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/crawl/Comm58.groovy | 3 +-- .../com/fangjia/script/temp/shenjian/crawl/CommFang.groovy | 2 +- .../com/fangjia/script/temp/shenjian/crawl/CommKe.groovy | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy index 3aa674810..caf289056 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy @@ -13,7 +13,7 @@ import org.apache.commons.lang3.time.DateUtils class Comm58 { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) - static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) static final def source = "58同城" static final def root = "58.com" @@ -43,7 +43,6 @@ class Comm58 { dbObject.put("city_short", cityShort) dbObject.put("district_id", id) dbObject.put("root", root) - dbObject.put("type", "comm") dbObject.put("c_date", c_date) dbObject.put("m_date", c_date) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy index e44632fc2..46659dc54 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -11,7 +11,7 @@ import org.apache.commons.lang3.time.DateUtils class CommFang { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) - static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) static void main(String[] args) { Date c_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy index 0e2baf552..0cf014627 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy @@ -11,7 +11,7 @@ import org.apache.commons.lang3.time.DateUtils class CommKe { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) - static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) static final def source = "贝壳找房" static final def root = "ke.com" From dfd5cf1e51ebc27a844f66f26faf9d361561617b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 8 May 2021 15:17:41 +0800 Subject: [PATCH 015/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=92=89=E9=92=89?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 12 +- .../shenjian/statistics/DingTalkUtils.groovy | 14 + .../temp/shenjian/statistics/Export.groovy | 51 +++ .../fangjia/script/statistics/ApiUtil.java | 305 ++++++++++++++++++ .../fangjia/script/statistics/Project.java | 92 ++++++ .../script/statistics/ProjectLabel.java | 57 ++++ .../com/fangjia/script/statistics/Task.java | 64 ++++ .../fangjia/script/statistics/TaskDetail.java | 67 ++++ .../fangjia/script/statistics/json/label.json | 38 +++ .../script/statistics/json/project.json | 225 +++++++++++++ .../fangjia/script/statistics/json/task.json | 85 +++++ .../script/statistics/json/taskDetail.json | 29 ++ 12 files changed, 1033 insertions(+), 6 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 1a14ef3ee..0d5adfa99 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -481,11 +481,11 @@ easyexcel 2.2.8 + + org.projectlombok + lombok + 1.18.20 + provided + - - - elastic.co - https://artifacts.elastic.co/maven - - diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy new file mode 100644 index 000000000..f49025b41 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy @@ -0,0 +1,14 @@ +/** + * 钉钉接口调用工具类 + * + * @author shenjian + * @since 2021/5/8 + */ +class DingTalkUtils { + + final String key = "ding0243a5c51f1f4aa8"; + + static void main(String[] args) { + + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy new file mode 100644 index 000000000..7f53ec256 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy @@ -0,0 +1,51 @@ +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel + +/** + * 绩效相关信息导出 + * + * @author shenjian + * @since 2021/5/8 + */ +class Export { + + static void main(String[] args) { + def result = [] + result << ['统计时间', '2021年4月1日-2021年4月30日'] + result << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] + + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[1] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[2] == null) { + ret[2] = label.getName() + } else { + ret[2] = ret[2] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + result << ret + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "项目统计"); + } +} + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java new file mode 100644 index 000000000..1c2836659 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -0,0 +1,305 @@ +package com.fangjia.script.statistics; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import org.apache.commons.codec.binary.Base64; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.DefaultHttpClient; + +import com.alibaba.fastjson.JSONObject; + +/** + * 铓傝殎鍒嗗伐寮�鏀惧钩鍙板伐鍏风被 + * 鐢变簬groovy璋冪敤浼氳鎶ョ鍚嶉敊璇棶棰橈紝鏁呯敤java鏂瑰紡 + */ +public class ApiUtil { + + private static final String secretKey = "HtRc7R5jI0plOOAukES4pM6yxWJbU9GyyM1xi4kH"; + private static final String corpId = "ding0243a5c51f1f4aa8"; + + public static String getApiSingnature(String body, Long timeStamp, String secretKey) throws Exception { + JSONObject bodyObj = JSONObject.parseObject(body); + return getApiSingnature(bodyObj, timeStamp, secretKey); + } + + /** + * 鑾峰彇绛惧悕 + */ + public static String getApiSingnature(JSONObject body, Long timeStamp, String secretKey) throws Exception { + String signature = ""; + try { + //鍙傛暟鎺掑簭锛屼繚璇佺鍚嶄竴鑷� + List fileds = new ArrayList<>(); + Set> set = body.entrySet(); + for (Entry entry : set) { + fileds.add(entry.getKey()); + } + //鍙傛暟鎵ц鎺掑簭 + Collections.sort(fileds); + Map params = new LinkedHashMap(); + for (String key : fileds) { + Object obj = body.get(key); + if(obj != null){ + params.put(key, obj); + } + } + String stringToSign = "timestamp="+timeStamp+"&body="+ params.toString(); + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secretKey.getBytes("UTF-8"), "HmacSHA256")); + byte[] signatureBytes = mac.doFinal(stringToSign.getBytes("UTF-8")); + signature = new String(Base64.encodeBase64(signatureBytes)); + return signature; + } catch (Exception e) { + e.printStackTrace(); + throw new Exception("璁$畻绛惧悕閿欒"); + } + } + + public static JSONObject post(String url, JSONObject json) throws Exception { + return (JSONObject) post(url, json, new JSONObject()); + } + + + private static Object post(String url, JSONObject json, Object t) throws Exception { + HttpClient client = new DefaultHttpClient(); + HttpPost post = new HttpPost(url); + + post.setHeader("Content-Type", "application/json"); + String result = ""; + try { + StringEntity s = new StringEntity(json.toJSONString(), "utf-8"); + post.setEntity(s); + + // 鍙戦�佽姹� + HttpResponse httpResponse = client.execute(post); + + // 鑾峰彇鍝嶅簲杈撳叆娴� + InputStream inStream = httpResponse.getEntity().getContent(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, "utf-8")); + StringBuilder strber = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null){ + strber.append(line + "\n"); + } + inStream.close(); + result = strber.toString(); + if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + return JSONObject.parseObject(result, t.getClass()); + } else { + return null; + } + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("璇锋眰寮傚父"); + } + } + + /** + * 鑾峰彇椤圭洰鏍囩 + * + * @return + */ + public static List listProjectLabel(String projectId) { + List labels = null; + try { + JSONObject body = new JSONObject(); + body.put("id", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/project/getProjectLabel", param); + if ((Integer)result.get("code") == 0) { + JSONArray jsonArray = result.getJSONArray("data"); + labels = jsonArray.toJavaList(ProjectLabel.class); + } + } catch (Exception e) { + e.printStackTrace(); + } + return labels; + } + + /** + * 鑾峰彇椤圭洰鍒楄〃 + * @return + */ + public static List listProject() { + List project = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("http://api.antbim.net/ant_api/project/getProjectList", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + project = jsonArray.toJavaList(Project.class); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return project; + } + + /** + * 鑾峰彇椤圭洰涓嬩换鍔℃竻鍗曞垪琛� + * + * @return + */ + public static JSONObject listInventory(String projectId) { + JSONObject result = new JSONObject(); + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + result = ApiUtil.post("https://api.antbim.net/ant_api/project/inventory/getInventoryList", param); + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + + + /** + * 鑾峰彇椤圭洰涓嬩换鍔$被鍨嬪垪琛� + * @return + */ + public static JSONObject listTaskType(String projectId) { + JSONObject result = new JSONObject(); + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + result = ApiUtil.post("https://api.antbim.net/ant_api/project/tasktype/getTasktypeList", param); + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + + /** + * 鑾峰彇椤圭洰鍐呬换鍔″垪琛� + * + * @return + */ + public static List listTaskByProject(String projectId, String startTime, String endTime) { + List tasks = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + body.put("createtimeStart", startTime); + body.put("createtimeEnd", endTime); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByProject", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + tasks = jsonArray.toJavaList(Task.class); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return tasks; + } + + /** + * 鑾峰彇浠诲姟璇︽儏 + * + * @return + */ + public static TaskDetail getTaskDetail(String taskId) { + TaskDetail detail = null; + try { + JSONObject body = new JSONObject(); + body.put("id", taskId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskDetail", param); + if ((Integer)result.get("code") == 0) { + JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data"))); + detail = obj.toJavaObject(TaskDetail.class); + } + } catch (Exception e) { + e.printStackTrace(); + } + return detail; + } + + public static void main(String[] args) { + //List projects = ApiUtil.listProject(); + //System.out.println(ApiUtil.listTaskByProject("ecd77309ab1c4b83b54ee87b8ddd4f98", "2021-04-01", "2021-04-30")); + System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java new file mode 100644 index 000000000..e5ea5b6a9 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java @@ -0,0 +1,92 @@ +package com.fangjia.script.statistics; + +/** + * @author shenjian + * @since 2021/5/8 + */ +public class Project { + private Integer taskCount; + private Integer taskendCount; + private String name; + private CreateUser createUser; + private String id; + private String state; + private String createDate; + + public Integer getTaskCount() { + return taskCount; + } + + public void setTaskCount(Integer taskCount) { + this.taskCount = taskCount; + } + + public Integer getTaskendCount() { + return taskendCount; + } + + public void setTaskendCount(Integer taskendCount) { + this.taskendCount = taskendCount; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public CreateUser getCreateUser() { + return createUser; + } + + public void setCreateUser(CreateUser createUser) { + this.createUser = createUser; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCreateDate() { + return createDate; + } + + public void setCreateDate(String createDate) { + this.createDate = createDate; + } +} + +class CreateUser { + private String name; + private String userid; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUserid() { + return userid; + } + + public void setUserid(String userid) { + this.userid = userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java new file mode 100644 index 000000000..bb5864998 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java @@ -0,0 +1,57 @@ +package com.fangjia.script.statistics; + +/** + * @author shenjian + * @since 2021/5/8 + */ +public class ProjectLabel { + + private String color; + private String name; + private Groups groups; + private String id; + + public String getColor() { + return color; + } + + public void setColor(String color) { + this.color = color; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Groups getGroups() { + return groups; + } + + public void setGroups(Groups groups) { + this.groups = groups; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} + +class Groups { + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java new file mode 100644 index 000000000..9a0c728c4 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java @@ -0,0 +1,64 @@ +package com.fangjia.script.statistics; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/8 + */ +@NoArgsConstructor +@Data +public class Task { + + @JSONField(name = "code") + private String code; + @JSONField(name = "response") + private ResponseDTO response; + @JSONField(name = "planBeginTime") + private String planBeginTime; + @JSONField(name = "name") + private String name; + @JSONField(name = "planEndTime") + private String planEndTime; + @JSONField(name = "id") + private String id; + @JSONField(name = "state") + private String state; + @JSONField(name = "inventory") + private InventoryDTO inventory; + @JSONField(name = "flow") + private FlowDTO flow; + @JSONField(name = "createDate") + private String createDate; + + @NoArgsConstructor + @Data + public static class ResponseDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } + + @NoArgsConstructor + @Data + public static class InventoryDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "id") + private String id; + @JSONField(name = "abbrev") + private String abbrev; + } + + @NoArgsConstructor + @Data + public static class FlowDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java new file mode 100644 index 000000000..3d0d4acfe --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java @@ -0,0 +1,67 @@ +package com.fangjia.script.statistics; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/8 + */ +@NoArgsConstructor +@Data +public class TaskDetail { + + @JSONField(name = "code") + private String code; + @JSONField(name = "realityBeginDate") + private String realityBeginDate; + @JSONField(name = "assignDate") + private String assignDate; + @JSONField(name = "flowConten") + private String flowConten; + @JSONField(name = "description") + private String description; + @JSONField(name = "realityEndDate") + private String realityEndDate; + @JSONField(name = "workTime") + private String workTime; + @JSONField(name = "confirmDate") + private String confirmDate; + @JSONField(name = "response") + private ResponseDTO response; + @JSONField(name = "name") + private String name; + @JSONField(name = "inventoryId") + private String inventoryId; + @JSONField(name = "state") + private String state; + @JSONField(name = "id") + private String id; + @JSONField(name = "planEndDate") + private String planEndDate; + @JSONField(name = "planBeginDate") + private String planBeginDate; + @JSONField(name = "flow") + private FlowDTO flow; + @JSONField(name = "createDate") + private String createDate; + + @NoArgsConstructor + @Data + public static class ResponseDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } + + @NoArgsConstructor + @Data + public static class FlowDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json new file mode 100644 index 000000000..10af777c0 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json @@ -0,0 +1,38 @@ +{ + "code": 0, + "data": [ + { + "color": "74CBFF", + "name": "建行项目", + "groups": { + "name": "项目类别" + }, + "id": "3538604d04b1495585a45c23379b65f5" + }, + { + "color": "75ccff", + "name": "丁世菁", + "groups": { + "name": "商务负责人" + }, + "id": "c9055e8370464537a2eefc3aa925097b" + }, + { + "color": "88C896", + "name": "准备中", + "groups": { + "name": "项目进展" + }, + "id": "6138228645e347b3a1f6ed52ece457c5" + }, + { + "color": "75ccff", + "name": "董彦均", + "groups": { + "name": "商务负责人" + }, + "id": "8100d58e9a3e4c8cb9be9406147ee93b" + } + ], + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json new file mode 100644 index 000000000..f3e658cca --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json @@ -0,0 +1,225 @@ +{ + "code": 0, + "data": { + "count": 18, + "list": [ + { + "taskCount": 2, + "taskendCount": 2, + "name": "建行配套数据采购项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "77478f60337840ccaf5ebb294ac1c987", + "state": "进行中", + "createDate": "2021-04-12 17:51:21" + }, + { + "taskCount": 3, + "taskendCount": 3, + "name": "房金部房产估值模型", + "createUser": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "id": "e7caf92da7be4c529c8733cbb196173e", + "state": "进行中", + "createDate": "2021-04-12 08:43:54" + }, + { + "taskCount": 1, + "taskendCount": 0, + "name": "地址标准化项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f7c307d9af794f4589f0a7027142e6e8", + "state": "进行中", + "createDate": "2021-04-01 10:03:56" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "企业经营地址库", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "59b1247428a445f5971f8f06091fb558", + "state": "进行中", + "createDate": "2021-03-22 15:10:41" + }, + { + "taskCount": 11, + "taskendCount": 8, + "name": "建信住房租金估价报告项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "ecd77309ab1c4b83b54ee87b8ddd4f98", + "state": "进行中", + "createDate": "2021-03-19 10:12:40" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "上数分地址标准化项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "4f22540647c74fc0a8277d798fbe9bfc", + "state": "进行中", + "createDate": "2021-03-18 16:50:19" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "建行开发商项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f82329c2390c4d2399f96ba21e92c950", + "state": "进行中", + "createDate": "2021-03-15 09:52:34" + }, + { + "taskCount": 9, + "taskendCount": 8, + "name": "房金部楼盘字典数据测试", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "457f11e89aad42b6a7bb289e8341ac04", + "state": "进行中", + "createDate": "2021-03-12 17:40:02" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "房价通", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "6a218757c86648559dacd9e4538d3f0a", + "state": "进行中", + "createDate": "2021-03-11 14:49:37" + }, + { + "taskCount": 1, + "taskendCount": 1, + "name": "亲邻科技", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "33c2a17a6d194168a2838c9ae4bd18b3", + "state": "进行中", + "createDate": "2021-03-10 11:46:36" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "福州农商银行", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f1f7bf085c3b4033932b8e78d2a644ed", + "state": "进行中", + "createDate": "2021-03-10 10:43:23" + }, + { + "taskCount": 27, + "taskendCount": 25, + "name": "房价网基础数据建设", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "ddd31486abb645eaa8a48fb43d57045b", + "state": "进行中", + "createDate": "2021-03-10 10:27:53" + }, + { + "taskCount": 1, + "taskendCount": 1, + "name": "江南农村商业银行", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "8f18fddec76a4ca788946825a741abdc", + "state": "进行中", + "createDate": "2021-03-09 09:38:15" + }, + { + "taskCount": 72, + "taskendCount": 65, + "name": "建行楼盘数据治理项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "41bdf23cbf634035b694f92ba72b067f", + "state": "进行中", + "createDate": "2021-03-02 16:54:38" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "江苏农行项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "a10e63ac24b9487cb1dd1140dd2ffc94", + "state": "进行中", + "createDate": "2021-03-02 09:08:45" + }, + { + "taskCount": 0, + "taskendCount": 0, + "name": "浙商银行估值项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "53aa253f216542dfb8022dcbd18601cc", + "state": "进行中", + "createDate": "2021-03-02 09:07:54" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "机房运维", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "fc76aa65155741c1a074305741a067eb", + "state": "进行中", + "createDate": "2021-03-02 09:01:42" + }, + { + "taskCount": 27, + "taskendCount": 26, + "name": "浙商银行本地化部署", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "71e520aff8e243f3b04e9d6fea7867d1", + "state": "进行中", + "createDate": "2021-03-01 16:36:41" + } + ] + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json new file mode 100644 index 000000000..8026c9d69 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json @@ -0,0 +1,85 @@ +{ + "code": 0, + "data": { + "count": 4, + "list": [ + { + "code": "5", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-06 09:00:00", + "name": "地图展示上海租金数据", + "planEndTime": "2021-04-06 18:00:00", + "id": "3bbb043ad0b44ea2a1213e725274e3ad", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-06 15:52:01" + }, + { + "code": "6", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-07 09:00:00", + "name": "租房报告预览下载;es迁移丢失数据同步", + "planEndTime": "2021-04-08 18:59:00", + "id": "8725262c17db408dab1f31473d50598f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-07 09:31:53" + }, + { + "code": "7", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-08 09:00:00", + "name": "根据新的租房报告进仁行接口修改", + "planEndTime": "2021-04-08 18:00:00", + "id": "309bdc4420e74868bda27884f4a46508", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-12 11:24:01" + }, + { + "code": "8", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-16 09:23:00", + "name": "估值页面报告优化及小学数据按标准导出", + "planEndTime": "2021-04-16 19:00:00", + "id": "45f9d101b18b48a08045de76cf2e53af", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-16 09:23:21" + } + ] + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json new file mode 100644 index 000000000..362cb7398 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json @@ -0,0 +1,29 @@ +{ + "code": 0, + "data": { + "code": "5", + "realityBeginDate": "2021-04-06 09:00:00", + "assignDate": "2021-04-06 15:52:01", + "flowConten": "1", + "description": "", + "realityEndDate": "2021-04-08 16:49:07", + "workTime": "1.00", + "confirmDate": "2021-04-08 16:49:07", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "name": "地图展示上海租金数据", + "inventoryId": "d00fcb9c5307499eab8f9798a23f6360", + "state": "已完成", + "id": "3bbb043ad0b44ea2a1213e725274e3ad", + "planEndDate": "2021-04-06 18:00:00", + "planBeginDate": "2021-04-06 09:00:00", + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-06 15:52:01" + }, + "message": "成功" +} From 17fa2bb110df6fc38ecf5fe65e7b203c29ca74d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 09:32:20 +0800 Subject: [PATCH 016/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/statistics/Export.groovy | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy index 7f53ec256..b8cb65980 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy @@ -1,6 +1,9 @@ import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils /** * 绩效相关信息导出 @@ -10,10 +13,13 @@ import com.fangjia.script.statistics.ProjectLabel */ class Export { + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-04-01" + static final String endTime = "2021-04-30" + static void main(String[] args) { + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] - result << ['统计时间', '2021年4月1日-2021年4月30日'] - result << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] List projects = ApiUtil.listProject(); for (Project project : projects) { @@ -31,10 +37,34 @@ class Export { ret[0] = label.getName() } } + // 获取项目总工时 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + Double totalTime = 0D; + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 先取绩效,不存在则取工时 + if (taskDetail.getFlowConten() != null) { + totalTime += Double.valueOf(taskDetail.getFlowConten()) + } else { + totalTime += Double.valueOf(taskDetail.getWorkTime()) + } + } + } + ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, totalTime) result << ret } + def sortedResult = [] + sortedResult << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] + for (def label : sort) { + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + } + } + } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "项目统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计"); } } From bf6af8637e8457a6797867695c8df8f1693b0c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 11:48:10 +0800 Subject: [PATCH 017/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=A9=E6=95=88?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{Export.groovy => ProjectExport.groovy} | 6 +- .../shenjian/statistics/ScoreExport.groovy | 113 ++++++++++++++++++ 2 files changed, 116 insertions(+), 3 deletions(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/{Export.groovy => ProjectExport.groovy} (96%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy similarity index 96% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index b8cb65980..76ea84abe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -6,12 +6,12 @@ import com.fangjia.script.statistics.TaskDetail import org.apache.commons.collections.CollectionUtils /** - * 绩效相关信息导出 + * 项目导出 * * @author shenjian * @since 2021/5/8 */ -class Export { +class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01" @@ -64,7 +64,7 @@ class Export { } } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计"); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy new file mode 100644 index 000000000..baead79bc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -0,0 +1,113 @@ +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils + +/** + * 绩效导出 + * + * @author shenjian + * @since 2021/5/10 + */ +class ScoreExport { + + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-04-01" + static final String endTime = "2021-04-30" + + static void main(String[] args) { + def sortedResult = [] + sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def result = [] + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[2] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[3] == null) { + ret[3] = label.getName() + } else { + ret[3] = ret[3] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + // 获取项目下任务 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + Double totalTime = 0D; + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true + } + } + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + } + } + } + result << ret + } + def allTotal = [] + for (def label : sort) { + def total = [] + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + for (int i = 4; i < ret.size(); i++) { + if (total[i] == null) { + total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i]) + } else { + total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i])) + } + } + } + } + + total[1] = 0 + total[2] = "小计" + sortedResult << total + for (int i = 4; i < total.size(); i++) { + if (allTotal[i] == null) { + allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i]) + } else { + allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + } + allTotal[2] = "人员绩效总计" + sortedResult << allTotal + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + } +} + + + + + + + + + + + + From ab7213dd148ffddb87c80544f5a587ed89434cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 14:36:21 +0800 Subject: [PATCH 018/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/TaskExport.groovy | 113 +++ .../fangjia/script/statistics/ApiUtil.java | 39 +- .../script/statistics/json/personTask.json | 928 ++++++++++++++++++ 3 files changed, 1079 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy new file mode 100644 index 000000000..f2e64dd9e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -0,0 +1,113 @@ +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils + +/** + * 任务统计 + * + * @author shenjian + * @since 2021/5/10 + */ +class TaskExport { + + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-04-01" + static final String endTime = "2021-04-30" + + static void main(String[] args) { + def sortedResult = [] + sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def result = [] + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[2] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[3] == null) { + ret[3] = label.getName() + } else { + ret[3] = ret[3] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + // 获取项目下任务 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + Double totalTime = 0D; + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true + } + } + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + } + } + } + result << ret + } + def allTotal = [] + for (def label : sort) { + def total = [] + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + for (int i = 4; i < ret.size(); i++) { + if (total[i] == null) { + total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i]) + } else { + total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i])) + } + } + } + } + + total[1] = 0 + total[2] = "小计" + sortedResult << total + for (int i = 4; i < total.size(); i++) { + if (allTotal[i] == null) { + allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i]) + } else { + allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + } + allTotal[2] = "人员绩效总计" + sortedResult << allTotal + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + } +} + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 1c2836659..33c96ae42 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -267,6 +267,42 @@ public class ApiUtil { return tasks; } + /** + * 鑾峰彇浜哄憳浠诲姟鍒楄〃 + */ + public static List listTaskByPerson(String startTime, String endTime) { + List tasks = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + //body.put("userid", ""); + body.put("type", ""); + body.put("timezone", ""); + body.put("createtimeStart", startTime); + body.put("createtimeEnd", endTime); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByPerson", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + tasks = jsonArray.toJavaList(Task.class); + } + } + } catch (Exception ex) { + ex.printStackTrace(); + } + return tasks; + } + /** * 鑾峰彇浠诲姟璇︽儏 * @@ -300,6 +336,7 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); //System.out.println(ApiUtil.listTaskByProject("ecd77309ab1c4b83b54ee87b8ddd4f98", "2021-04-01", "2021-04-30")); - System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + System.out.println(ApiUtil.listTaskByPerson("2021-04-01", "2021-04-30")); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json new file mode 100644 index 000000000..d7510e331 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json @@ -0,0 +1,928 @@ +{ + "code": 0, + "data": { + "count": 176, + "list": [ + { + "code": "78", + "response": { + "name": "黄春英", + "userid": "143809622239903024" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "小区别名校验", + "planEndTime": "2021-05-14 23:59:00", + "id": "5c8f51d3971241eea4b1efc92b20fe5c", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 10:53:51" + }, + { + "code": "77", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-05-15 23:59:00", + "id": "3e12b0053aca4334ad28cd9bc30679f0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 10:19:16" + }, + { + "code": "76", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-05-15 23:59:00", + "id": "d6ccafed4b284951a5848ba62cfc00f0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:23:12" + }, + { + "code": "75", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "建行表整理及系统测试", + "planEndTime": "2021-05-15 23:59:00", + "id": "506a33123d984df98a05973fcc76f3d0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:12:20" + }, + { + "code": "74", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "系统测试", + "planEndTime": "2021-05-15 23:59:00", + "id": "bb3f9d174f9341b58b89203b3b326b9f", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:11:22" + }, + { + "code": "73", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-05-10 08:45:00", + "name": "导出小区别名数据给到人工检验,并整理人工反馈的小区别名校验结果;\n数据治理模型上线相关工作;", + "planEndTime": "2021-05-15 23:59:00", + "id": "ca1bae5186164b65b0807b597634317e", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 08:32:43" + }, + { + "code": "1.1", + "response": { + "name": "王辰岑", + "userid": "02245922215748" + }, + "planBeginTime": "2021-05-08 09:30:00", + "name": "网站注册及申请入驻供应商", + "planEndTime": "2021-05-08 18:00:00", + "id": "b1753f50c7a149f3a85c3a2fa0f180a8", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "7f01818d37c84c4d9337ef288623f5eb", + "abbrev": "A" + }, + "createDate": "2021-05-08 17:52:10" + }, + { + "code": "1", + "response": { + "name": "王辰岑", + "userid": "02245922215748" + }, + "name": "平台申请", + "id": "d357543551d14b22a91d2bd93b7afd31", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "7f01818d37c84c4d9337ef288623f5eb", + "abbrev": "A" + }, + "flow": { + "name": "蒋晓春", + "userid": "02245922395062" + }, + "createDate": "2021-05-08 17:49:05" + }, + { + "code": "11", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-05-08 09:00:00", + "name": "租金估值权限逻辑修改", + "planEndTime": "2021-05-08 19:00:00", + "id": "d0f6f2584e7440739c5b603f942c2bb7", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-05-08 09:42:10" + }, + { + "code": "27", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-06 09:00:00", + "name": "ES机器学习三台节点搭建", + "planEndTime": "2021-04-06 18:59:00", + "id": "a725c9be2bb2446eb180129fb58094d5", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:57:54" + }, + { + "code": "72", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "建行系统测试", + "planEndTime": "2021-05-08 23:59:00", + "id": "0c938c9b36054c84bc69ba166b31dea2", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:55:35" + }, + { + "code": "71", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "数据融合人工反馈处理", + "planEndTime": "2021-05-08 23:59:00", + "id": "daa8d8e8a69842eaae621a33e8690e55", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:43:55" + }, + { + "code": "26", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-05-07 09:00:00", + "name": "分析爬虫抓取情况,导出亲邻数据,导出四川锦城数据", + "planEndTime": "2021-05-07 18:00:00", + "id": "1c9085be784542fb917a2ff2767388c9", + "state": "待确认", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-05-07 15:39:42" + }, + { + "code": "70", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-07 00:00:00", + "name": "新增安居客小区详情解析,优化安居客小区详情抓取", + "id": "e7717e32c9864511aabfedd41117d6ca", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 14:37:30" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "微元素帐号整理", + "id": "4de48c77988743a5ab83343b40adee97", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "907da15cb4a54184bff9b32345300d2a", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:54:24" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 14:00:00", + "name": "公司名称及地址匹配", + "id": "286ad20909ca4b27bd70c34eeea6a3c1", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "270452cd5725475095c7e668ea137481", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:22:42" + }, + { + "code": "69", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 09:30:00", + "name": "新增搜房小区列表抓取及解析\n导出新租金米数据", + "planEndTime": "2021-05-06 14:00:00", + "id": "35df605842d941a58b94d6b5e3c8e5b7", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:21:21" + }, + { + "code": "68", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-05-08 23:59:00", + "id": "e3758204453b4858a02c33f7f0ceaf45", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 09:41:20" + }, + { + "code": "67", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-05-08 23:59:00", + "id": "613d42946b694b24b9c6d29a7fb3cbc8", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 09:41:00" + }, + { + "code": "66", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-05-06 08:45:00", + "name": "1.将小区别名数据导出给人工校验2.将78数据库中的人工标注数据,经纬度围栏数据放到85、88数据库", + "planEndTime": "2021-05-08 23:59:00", + "id": "2410d7840bff48af8b2202c7c7271cfb", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 08:30:36" + }, + { + "code": "10", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-05 00:00:00", + "name": "优化诸葛找房抓取稠租房少的问题\n新增搜房通过入小区页面抓取租房户房源\n及验证曹老师说抓取少的网站重抓", + "planEndTime": "2021-04-07 23:59:00", + "id": "c37324ae475149cfb0648731934873a9", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:35:09" + }, + { + "code": "25", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-29 00:00:00", + "name": "网站种子配置及福州二手房抓取", + "planEndTime": "2021-04-30 23:59:00", + "id": "a1326b626acf444193969c755287de1c", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:32:07" + }, + { + "code": "9", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-03 00:00:00", + "name": "新增我爱我家,百姓,赶集的抓取解析,\n新增我爱我家成交抓取,\n修改诸葛找房抓取,解析", + "planEndTime": "2021-04-03 23:59:00", + "id": "73e49abc26574e78b696113bcf200019", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:10:56" + }, + { + "code": "65", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "建行数据融合统计结果(彦均)", + "planEndTime": "2021-04-25 23:59:00", + "id": "be85282dc57d46e3a36fc0801107e605", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:03:21" + }, + { + "code": "3", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "建行数据导出(彦均)\n房源量小于20的小区\n小区统计", + "planEndTime": "2021-04-19 23:59:00", + "id": "af3c862c7c674f97855061cb923fbfef", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "ac8c4d70ea3842c2a6c628a64628989a", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:00:56" + }, + { + "code": "24", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-29 00:00:00", + "name": "通过小区抓安居客房源,配完,反爬厉害\n分析抓取现状\n总结输出房源方案", + "planEndTime": "2021-04-29 23:59:00", + "id": "bfccb3c436af4d1a9263ae4879dc7fee", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-29 18:29:05" + }, + { + "code": "1", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "企址库抓取\n尝试国家信息,天眼查,爱企查有js问题\n个企查查,启信宝有豋录问题,88不全,最后抓顺企", + "planEndTime": "2021-04-22 23:59:00", + "id": "b495c086d3fa4a6d8e6d7319fdef303f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "270452cd5725475095c7e668ea137481", + "abbrev": "A" + }, + "createDate": "2021-04-29 12:05:12" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-26 00:00:00", + "name": "建行案例测试批量评估价格,并并导出房源与神经网络j小区", + "planEndTime": "2021-04-28 23:59:00", + "id": "68afd0578fab469d8371fb510c1f8af9", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "ac8c4d70ea3842c2a6c628a64628989a", + "abbrev": "A" + }, + "createDate": "2021-04-29 11:51:14" + }, + { + "code": "26", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-26 00:00:00", + "name": "浙商本地化部署项目验证环境发布", + "planEndTime": "2021-04-28 23:59:00", + "id": "91ea49c8f5e747b18660dec02da7bfeb", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "b4abc1f7cad142c4be5e5b301ae08190", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-29 09:44:24" + }, + { + "code": "25", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-26 09:00:00", + "name": "浙商本地化验证环境部署工作", + "planEndTime": "2021-04-28 17:59:00", + "id": "09c28bed9a43475c9bd24a69066170d4", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "b4abc1f7cad142c4be5e5b301ae08190", + "abbrev": "A" + }, + "createDate": "2021-04-26 11:34:36" + }, + { + "code": "23", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "Python工具包封装", + "planEndTime": "2021-04-25 23:59:00", + "id": "f3a00c2e64184796a4a12ab37ce60ac5", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-25 18:24:34" + }, + { + "code": "64", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "系统测试,数据统计", + "planEndTime": "2021-04-30 23:59:00", + "id": "8b8be8a4a3424ac9a3cd74145d02f26f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 09:27:25" + }, + { + "code": "63", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-04-30 23:59:00", + "id": "1140039f97da40639763101258f4c300", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 09:14:40" + }, + { + "code": "62", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-30 23:59:00", + "id": "9c2064b98c814b3183189c6d6a68a068", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:41:04" + }, + { + "code": "61", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "96f4d6e9851b44fc84eb20a8fd58504a", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:39:14" + }, + { + "code": "60", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-04-25 08:45:00", + "name": "1.在82.83.84服务器上运行测试代码;\n2.编写”数据质量监测维度信息统计”的代码", + "planEndTime": "2021-04-30 23:59:00", + "id": "a354e6df53e34a9581fa5988f4334dfa", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:28:25" + }, + { + "code": "22", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "租金模型开发", + "planEndTime": "2021-04-27 23:59:00", + "id": "3f569aa260b04da88224834d56f43427", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-25 07:58:48" + }, + { + "code": "21", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "pyspider新增获取小区id接口及项目爬取时间监控", + "planEndTime": "2021-04-22 23:59:00", + "id": "f8122a43929646e8acfdbbf70091bf3c", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-21 09:44:31" + }, + { + "code": "59", + "response": { + "name": "黄春英", + "userid": "143809622239903024" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "别名表校验", + "planEndTime": "2021-04-25 23:59:00", + "id": "6a26dbc7534240f88861873177bca3ee", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-20 09:02:45" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "深圳房价角通室厅必填\n从室号解析楼层", + "planEndTime": "2021-04-19 18:00:00", + "id": "b4001b1b33fe486d93897d737dd950a3", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "cc6b1205b3d547f5867f44ddf5d568df", + "abbrev": "A" + }, + "createDate": "2021-04-19 19:23:25" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-13 00:00:00", + "name": "建行学校抓取合肥,苏苏州,西安", + "planEndTime": "2021-04-16 23:59:00", + "id": "e26d64fb288a4558bec894035db70f27", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "0fbd26a5a4914d92802352502819abd8", + "abbrev": "A" + }, + "createDate": "2021-04-19 19:14:47" + }, + { + "code": "58", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "建行数据系统开发", + "planEndTime": "2021-04-23 23:59:00", + "id": "de19c283d30843c9971d1eb5ba4aa157", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 16:14:04" + }, + { + "code": "20", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "修改均价测试系统bug", + "planEndTime": "2021-04-19 18:00:00", + "id": "d6f6e37425d24b2ba04ef1ce64958358", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-19 13:29:35" + }, + { + "code": "57", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "数据清洗", + "planEndTime": "2021-04-24 23:59:00", + "id": "a5d3e59e2fae4f42bc5097d50096edab", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:19:07" + }, + { + "code": "56", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "8ef405124f054f578fa4b21c7863c415", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:18:50" + }, + { + "code": "55", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "51d0a97ac69b48c89926ee7f2736e62d", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:06:04" + }, + { + "code": "54", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "楼盘治理上线联调", + "planEndTime": "2021-04-23 23:59:00", + "id": "d08c7c2c852048aebc8aa59d48cfbd21", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 08:43:11" + }, + { + "code": "53", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-04-19 08:45:00", + "name": "1.根据人工融合校验,优化清洗关键字和规则;2.在Linux的82.83.84服务器上测试代码", + "planEndTime": "2021-04-23 23:59:00", + "id": "2af0fe9397094718941cd081e5dfbe53", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 08:26:10" + }, + { + "code": "19", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "友达二手成交数据导入", + "planEndTime": "2021-04-19 18:00:00", + "id": "3a1891dfc95c4d39ac33ce64761a4a57", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-19 00:04:47" + }, + { + "code": "18", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-16 00:00:00", + "name": "深圳指导价入库与神经网络价格修复", + "planEndTime": "2021-04-16 23:59:00", + "id": "4999543a15ff429a85875348df048726", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-16 18:59:31" + } + ] + }, + "message": "成功" +} From 88211095fdc0167ad7c6af9cf5090e3197a6eeb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 16:16:53 +0800 Subject: [PATCH 019/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=9F=8E=E5=B8=82?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/crawl/City58.groovy | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy new file mode 100644 index 000000000..78f033fd4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy @@ -0,0 +1,277 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 58城市简写 + * + * @author shenjian + * @since 2021/5/10 + */ +class City58 { + + static String cityList = "{\n" + + " \"安徽\":{\n" + + " \"合肥\":\"hf|837\",\"芜湖\":\"wuhu|2045\",\"蚌埠\":\"bengbu|3470\",\"阜阳\":\"fy|2325\",\"淮南\":\"hn|2319\",\n" + + " \"安庆\":\"anqing|3251\",\"宿州\":\"suzhou|3359\",\"六安\":\"la|2328\",\"淮北\":\"huaibei|9357\",\"滁州\":\"chuzhou|10266\",\n" + + " \"马鞍山\":\"mas|2039\",\"铜陵\":\"tongling|10285\",\"宣城\":\"xuancheng|5633\",\"亳州\":\"bozhou|2329\",\"黄山\":\"huangshan|2323\",\n" + + " \"池州\":\"chizhou|10260\",\"巢湖\":\"ch|10229\",\"和县\":\"hexian|10892\",\"霍邱\":\"hq|11226\",\"桐城\":\"tongcheng|11296\",\n" + + " \"宁国\":\"ningguo|5645\",\"天长\":\"tianchang|10273\",\"东至\":\"dongzhi|10262\",\"无为\":\"wuweixian|10232\"\n" + + " }\n" + + " ,\n" + + " \"福建\":{\n" + + " \"福州\":\"fz|304\",\"厦门\":\"xm|606\",\"泉州\":\"qz|291\",\"莆田\":\"pt|2429\",\"漳州\":\"zhangzhou|710\",\n" + + " \"宁德\":\"nd|7951\",\"三明\":\"sm|2048\",\"南平\":\"np|10291\",\"龙岩\":\"ly|6752\",\"武夷山\":\"wuyishan|10761\",\n" + + " \"石狮\":\"shishi|296\",\"晋江\":\"jinjiangshi|297\",\"南安\":\"nananshi|293\",\"龙海\":\"longhai|713\",\"上杭\":\"shanghangxian|6757\",\n" + + " \"福安\":\"fuanshi|7969\",\"福鼎\":\"fudingshi|7970\",\"安溪\":\"anxixian|7100\",\"永春\":\"yongchunxian|7101\",\"永安\":\"yongan|2133\",\n" + + " \"漳浦\":\"zhangpu|717\"\n" + + " }\n" + + " ,\n" + + " \"广东\":{\n" + + " \"深圳\":\"sz|4\",\"广州\":\"gz|3\",\"东莞\":\"dg|413\",\"佛山\":\"fs|222\",\"中山\":\"zs|771\",\n" + + " \"珠海\":\"zh|910\",\"惠州\":\"huizhou|722\",\"江门\":\"jm|629\",\"汕头\":\"st|783\",\"湛江\":\"zhanjiang|791\",\n" + + " \"肇庆\":\"zq|901\",\"茂名\":\"mm|679\",\"揭阳\":\"jy|927\",\"梅州\":\"mz|9389\",\"清远\":\"qingyuan|7303\",\n" + + " \"阳江\":\"yj|2284\",\"韶关\":\"sg|2192\",\"河源\":\"heyuan|10467\",\"云浮\":\"yf|10485\",\"汕尾\":\"sw|9449\",\n" + + " \"潮州\":\"chaozhou|10461\",\"台山\":\"taishan|11263\",\"阳春\":\"yangchun|8566\",\"顺德\":\"sd|8716\",\"惠东\":\"huidong|725\",\n" + + " \"博罗\":\"boluo|726\",\"海丰\":\"haifengxian|9444\",\"开平\":\"kaipingshi|634\",\"陆丰\":\"lufengshi|9456\"\n" + + " }\n" + + " ,\n" + + " \"广西\":{\n" + + " \"南宁\":\"nn|845\",\"柳州\":\"liuzhou|7133\",\"桂林\":\"gl|1039\",\"玉林\":\"yulin|2337\",\"梧州\":\"wuzhou|2046\",\n" + + " \"北海\":\"bh|10536\",\"贵港\":\"gg|6770\",\"钦州\":\"qinzhou|2335\",\"百色\":\"baise|10513\",\"河池\":\"hc|2340\",\n" + + " \"来宾\":\"lb|10552\",\"贺州\":\"hezhou|10549\",\"防城港\":\"fcg|10539\",\"崇左\":\"chongzuo|10524\",\"桂平\":\"guipingqu|6774\",\n" + + " \"北流\":\"beiliushi|9168\",\"博白\":\"bobaixian|9173\",\"岑溪\":\"cenxi|2119\"\n" + + " }\n" + + " ,\n" + + " \"贵州\":{\n" + + " \"贵阳\":\"gy|2015\",\"遵义\":\"zunyi|7620\",\"黔东南\":\"qdn|9363\",\"黔南\":\"qn|10492\",\"六盘水\":\"lps|10506\",\n" + + " \"毕节\":\"bijie|10564\",\"铜仁\":\"tr|10417\",\"安顺\":\"anshun|7468\",\"黔西南\":\"qxn|10434\",\"仁怀\":\"renhuaishi|7628\",\n" + + " \"清镇\":\"qingzhen|12703\"\n" + + " }\n" + + " ,\n" + + " \"甘肃\":{\n" + + " \"兰州\":\"lz|952\",\"天水\":\"tianshui|8601\",\"白银\":\"by|10304\",\"庆阳\":\"qingyang|10475\",\"平凉\":\"pl|7154\",\n" + + " \"酒泉\":\"jq|10387\",\"张掖\":\"zhangye|10454\",\"武威\":\"wuwei|10448\",\"定西\":\"dx|10322\",\"金昌\":\"jinchang|7428\",\n" + + " \"陇南\":\"ln|10415\",\"临夏\":\"linxia|7112\",\"嘉峪关\":\"jyg|10362\",\"甘南\":\"gn|10343\",\"敦煌\":\"dunhuang|10390\"\n" + + " }\n" + + " ,\n" + + " \"海南\":{\n" + + " \"海口\":\"haikou|2053\",\"三亚\":\"sanya|2422\",\"五指山\":\"wzs|9952\",\"三沙\":\"sansha|13722\",\"琼海\":\"qh|10136\",\n" + + " \"文昌\":\"wenchang|9984\",\"万宁\":\"wanning|10022\",\"屯昌\":\"tunchang|10044\",\"琼中\":\"qiongzhong|10064\",\"陵水\":\"lingshui|10184\",\n" + + " \"东方\":\"df|10250\",\"定安\":\"da|10303\",\"澄迈\":\"cm|10331\",\"保亭\":\"baoting|10367\",\"白沙\":\"baish|10380\",\n" + + " \"儋州\":\"danzhou|10394\"\n" + + " }\n" + + " ,\n" + + " \"河南\":{\n" + + " \"郑州\":\"zz|342\",\"洛阳\":\"luoyang|556\",\"新乡\":\"xx|1016\",\"南阳\":\"ny|592\",\"许昌\":\"xc|977\",\n" + + " \"平顶山\":\"pds|1005\",\"安阳\":\"ay|1096\",\"焦作\":\"jiaozuo|3266\",\"商丘\":\"sq|1029\",\"开封\":\"kaifeng|2342\",\n" + + " \"濮阳\":\"puyang|2346\",\"周口\":\"zk|933\",\"信阳\":\"xy|8694\",\"驻马店\":\"zmd|1067\",\"漯河\":\"luohe|2347\",\n" + + " \"三门峡\":\"smx|9317\",\"鹤壁\":\"hb|2344\",\"济源\":\"jiyuan|9918\",\"明港\":\"mg|8541\",\"鄢陵\":\"yanling|9123\",\n" + + " \"禹州\":\"yuzhou|979\",\"长葛\":\"changge|9344\",\"灵宝\":\"lingbaoshi|9307\",\"杞县\":\"qixianqu|7389\",\"汝州\":\"ruzhou|1010\",\n" + + " \"项城\":\"xiangchengshi|935\",\"偃师\":\"yanshiqu|7121\",\"长垣\":\"changyuan|5936\",\"滑县\":\"huaxian|5405\",\"林州\":\"linzhou|1101\",\n" + + " \"沁阳\":\"qinyang|3268\",\"孟州\":\"mengzhou|3267\",\"温县\":\"wenxian|7312\",\"尉氏\":\"weishixian|7391\",\"兰考\":\"lankaoxian|7393\",\n" + + " \"通许\":\"tongxuxian|7390\",\"新安\":\"lyxinan|11217\",\"伊川\":\"yichuan|11220\",\"孟津\":\"mengjinqu|7122\",\"宜阳\":\"lyyiyang|11219\",\n" + + " \"舞钢\":\"wugang|1011\",\"永城\":\"yongcheng|1032\",\"睢县\":\"suixian|1038\",\"鹿邑\":\"luyi|939\",\"渑池\":\"yingchixian|9322\",\n" + + " \"沈丘\":\"shenqiu|942\",\"太康\":\"taikang|938\",\"商水\":\"shangshui|936\",\"淇县\":\"qixianq|9186\",\"浚县\":\"junxian|9185\",\n" + + " \"范县\":\"fanxian|7285\",\"固始\":\"gushixian|8698\",\"淮滨\":\"huaibinxian|8702\",\"邓州\":\"dengzhou|595\",\"新野\":\"xinye|603\"\n" + + " }\n" + + " ,\n" + + " \"黑龙江\":{\n" + + " \"哈尔滨\":\"hrb|202\",\"大庆\":\"dq|375\",\"齐齐哈尔\":\"qqhr|5853\",\"牡丹江\":\"mdj|3489\",\"绥化\":\"suihua|6718\",\n" + + " \"佳木斯\":\"jms|6776\",\"鸡西\":\"jixi|7289\",\"双鸭山\":\"sys|9837\",\"鹤岗\":\"hegang|9061\",\"黑河\":\"heihe|9862\",\n" + + " \"伊春\":\"yich|9773\",\"七台河\":\"qth|9848\",\"大兴安岭\":\"dxal|9878\",\"安达\":\"shanda|6720\",\"肇东\":\"shzhaodong|6721\",\n" + + " \"肇州\":\"zhaozhou|382\"\n" + + " }\n" + + " ,\n" + + " \"湖北\":{\n" + + " \"武汉\":\"wh|158\",\"宜昌\":\"yc|858\",\"襄阳\":\"xf|891\",\"荆州\":\"jingzhou|3479\",\"十堰\":\"shiyan|2032\",\n" + + " \"黄石\":\"hshi|1734\",\"孝感\":\"xiaogan|3434\",\"黄冈\":\"hg|2299\",\"恩施\":\"es|2302\",\"荆门\":\"jingmen|2296\",\n" + + " \"咸宁\":\"xianning|9617\",\"鄂州\":\"ez|9709\",\"随州\":\"suizhou|9656\",\"潜江\":\"qianjiang|9669\",\"天门\":\"tm|9517\",\n" + + " \"仙桃\":\"xiantao|9736\",\"神农架\":\"snj|9605\",\"宜都\":\"yidou|864\",\"汉川\":\"hanchuan|3439\",\"枣阳\":\"zaoyang|896\",\n" + + " \"武穴\":\"wuxueshi|7362\",\"钟祥\":\"zhongxiangshi|9119\",\"京山\":\"jingshanxian|9117\",\"沙洋\":\"shayangxian|9118\",\"松滋\":\"songzi|3484\",\n" + + " \"广水\":\"guangshuishi|9657\",\"赤壁\":\"chibishi|9623\",\"老河口\":\"laohekou|895\",\"谷城\":\"gucheng|899\",\"宜城\":\"yichengshi|897\",\n" + + " \"南漳\":\"nanzhang|898\",\"云梦\":\"yunmeng|3438\",\"安陆\":\"anlu|3442\",\"大悟\":\"dawu|3437\",\"孝昌\":\"xiaochang|3436\",\n" + + " \"当阳\":\"dangyang|865\",\"枝江\":\"zhijiang|866\",\"嘉鱼\":\"jiayuxian|9624\",\"随县\":\"suixia|9660\"\n" + + " }\n" + + " ,\n" + + " \"湖南\":{\n" + + " \"长沙\":\"cs|414\",\"株洲\":\"zhuzhou|1086\",\"益阳\":\"yiyang|10198\",\"常德\":\"changde|872\",\"衡阳\":\"hy|914\",\n" + + " \"湘潭\":\"xiangtan|2047\",\"岳阳\":\"yy|821\",\"郴州\":\"chenzhou|5695\",\"邵阳\":\"shaoyang|2303\",\"怀化\":\"hh|5756\",\n" + + " \"永州\":\"yongzhou|2307\",\"娄底\":\"ld|9481\",\"湘西\":\"xiangxi|10219\",\"张家界\":\"zjj|6788\",\"醴陵\":\"liling|1091\",\n" + + " \"澧县\":\"lixian|876\",\"桂阳\":\"czguiyang|5699\",\"资兴\":\"zixing|5698\",\"永兴\":\"yongxing|5701\",\"常宁\":\"changningshi|921\",\n" + + " \"祁东\":\"qidongxian|5690\",\"衡东\":\"hengdong|5693\",\"冷水江\":\"lengshuijiangshi|9470\",\"涟源\":\"lianyuanshi|9471\",\"双峰\":\"shuangfengxian|9473\",\n" + + " \"邵阳县\":\"shaoyangxian|6955\",\"邵东\":\"shaodongxian|6954\",\"沅江\":\"yuanjiangs|10201\",\"南县\":\"nanxian|10202\",\"祁阳\":\"qiyang|8532\",\n" + + " \"湘阴\":\"xiangyin|828\",\"华容\":\"huarong|830\",\"慈利\":\"cilixian|6791\",\"攸县\":\"zzyouxian|1095\"\n" + + " }\n" + + " ,\n" + + " \"河北\":{\n" + + " \"石家庄\":\"sjz|241\",\"保定\":\"bd|424\",\"唐山\":\"ts|276\",\"廊坊\":\"lf|772\",\"邯郸\":\"hd|572\",\n" + + " \"秦皇岛\":\"qhd|1078\",\"沧州\":\"cangzhou|652\",\"邢台\":\"xt|751\",\"衡水\":\"hs|993\",\"张家口\":\"zjk|3328\",\n" + + " \"承德\":\"chengde|6760\",\"定州\":\"dingzhou|8398\",\"馆陶\":\"gt|8706\",\"张北\":\"zhangbei|11201\",\"赵县\":\"zx|9048\",\n" + + " \"正定\":\"zd|3198\",\"迁安市\":\"qianan|284\",\"任丘\":\"renqiu|656\",\"三河\":\"sanhe|776\",\"武安\":\"wuan|577\",\n" + + " \"雄安新区\":\"xionganxinqu|111234\",\"燕郊\":\"lfyanjiao|12804\",\"涿州\":\"zhuozhou|428\",\"河间\":\"hejian|658\",\"黄骅\":\"huanghua|657\",\n" + + " \"沧县\":\"cangxian|659\",\"磁县\":\"cixian|591\",\"涉县\":\"shexian|14059\",\"霸州\":\"bazhou|775\",\"香河\":\"xianghe|5395\",\n" + + " \"固安\":\"lfguan|12803\",\"遵化市\":\"zunhua|283\",\"迁西\":\"qianxixian|7061\",\"玉田\":\"yutianxian|7060\",\"滦南\":\"luannanxian|7066\",\n" + + " \"沙河\":\"shaheshi|755\"\n" + + " }\n" + + " ,\n" + + " \"江苏\":{\n" + + " \"苏州\":\"su|5\",\"南京\":\"nj|172\",\"无锡\":\"wx|93\",\"常州\":\"cz|463\",\"徐州\":\"xz|471\",\n" + + " \"南通\":\"nt|394\",\"扬州\":\"yz|637\",\"盐城\":\"yancheng|613\",\"淮安\":\"ha|968\",\"连云港\":\"lyg|2049\",\n" + + " \"泰州\":\"taizhou|693\",\"宿迁\":\"suqian|2350\",\"镇江\":\"zj|645\",\"沭阳\":\"shuyang|5772\",\"大丰\":\"dafeng|11279\",\n" + + " \"如皋\":\"rugao|397\",\"启东\":\"qidong|400\",\"溧阳\":\"liyang|469\",\"海门\":\"haimen|399\",\"东海\":\"donghai|2147\",\n" + + " \"扬中\":\"yangzhong|649\",\"兴化\":\"xinghuashi|699\",\"新沂\":\"xinyishi|478\",\"泰兴\":\"taixing|696\",\"如东\":\"rudong|402\",\n" + + " \"邳州\":\"pizhou|477\",\"沛县\":\"xzpeixian|11349\",\"靖江\":\"jingjiang|698\",\"建湖\":\"jianhu|618\",\"海安\":\"haian|401\",\n" + + " \"东台\":\"dongtai|615\",\"丹阳\":\"danyang|648\",\"宝应县\":\"baoyingx|14451\",\"灌南\":\"guannan|2150\",\"灌云\":\"guanyun|2148\",\n" + + " \"姜堰\":\"jiangyan|697\",\"金坛\":\"jintan|468\",\"昆山\":\"szkunshan|16\",\"泗洪\":\"sihong|5958\",\"泗阳\":\"siyang|5959\",\n" + + " \"句容\":\"jurong|650\",\"射阳\":\"sheyang|621\",\"阜宁\":\"funingxian|620\",\"响水\":\"xiangshui|619\",\"盱眙\":\"xuyi|976\",\n" + + " \"金湖\":\"jinhu|975\",\"江阴\":\"jiangyins|34984\"\n" + + " }\n" + + " ,\n" + + " \"江西\":{\n" + + " \"南昌\":\"nc|669\",\"赣州\":\"ganzhou|2363\",\"九江\":\"jj|2247\",\"宜春\":\"yichun|5709\",\"吉安\":\"ja|2364\",\n" + + " \"上饶\":\"sr|10120\",\"萍乡\":\"px|2248\",\"抚州\":\"fuzhou|10134\",\"景德镇\":\"jdz|2360\",\"新余\":\"xinyu|10115\",\n" + + " \"鹰潭\":\"yingtan|3209\",\"永新\":\"yxx|11077\",\"乐平\":\"lepingshi|9048\",\"进贤\":\"jinxian|677\",\"分宜\":\"fenyi|10118\",\n" + + " \"丰城\":\"fengchengshi|5711\",\"樟树\":\"zhangshu|5713\",\"高安\":\"gaoan|5712\",\"余江\":\"yujiang|3210\",\"南城\":\"nanchengx|10137\",\n" + + " \"浮梁\":\"fuliangxian|9071\"\n" + + " }\n" + + " ,\n" + + " \"吉林\":{\n" + + " \"长春\":\"cc|319\",\"吉林\":\"jl|700\",\"四平\":\"sp|10171\",\"延边\":\"yanbian|3184\",\"松原\":\"songyuan|2315\",\n" + + " \"白城\":\"bc|5918\",\"通化\":\"th|10159\",\"白山\":\"baishan|10179\",\"辽源\":\"liaoyuan|2501\",\"公主岭\":\"gongzhuling|10171\",\n" + + " \"梅河口\":\"meihekou|10162\",\"扶余\":\"fuyuxian|9085\",\"长岭\":\"changlingxian|9084\",\"桦甸\":\"huadian|706\",\"磐石\":\"panshi|708\",\n" + + " \"梨树县\":\"lishu|10176\"\n" + + " }\n" + + " ,\n" + + " \"辽宁\":{\n" + + " \"沈阳\":\"sy|188\",\"大连\":\"dl|147\",\"鞍山\":\"as|523\",\"锦州\":\"jinzhou|2354\",\"抚顺\":\"fushun|5722\",\n" + + " \"营口\":\"yk|5898\",\"盘锦\":\"pj|2041\",\"朝阳\":\"cy|10106\",\"丹东\":\"dandong|3445\",\"辽阳\":\"liaoyang|2038\",\n" + + " \"本溪\":\"benxi|5845\",\"葫芦岛\":\"hld|10088\",\"铁岭\":\"tl|6729\",\"阜新\":\"fx|10097\",\"庄河\":\"pld|3306\",\n" + + " \"瓦房店\":\"wfd|3279\",\"灯塔\":\"dengta|2071\",\"凤城\":\"fengcheng|3450\",\"北票\":\"beipiao|10109\",\"开原\":\"kaiyuan|6733\"\n" + + " }\n" + + " ,\n" + + " \"宁夏\":{\n" + + " \"银川\":\"yinchuan|2054\",\"吴忠\":\"wuzhong|9962\",\"石嘴山\":\"szs|9971\",\"中卫\":\"zw|9951\",\"固原\":\"guyuan|2421\"\n" + + " }\n" + + " ,\n" + + " \"内蒙古\":{\n" + + " \"呼和浩特\":\"hu|811\",\"包头\":\"bt|801\",\"赤峰\":\"chifeng|6700\",\"鄂尔多斯\":\"erds|2037\",\"通辽\":\"tongliao|10015\",\n" + + " \"呼伦贝尔\":\"hlbe|10039\",\"巴彦淖尔市\":\"bycem|10070\",\"乌兰察布\":\"wlcb|9993\",\"锡林郭勒\":\"xl|2408\",\"兴安盟\":\"xam|9976\",\n" + + " \"乌海\":\"wuhai|2404\",\"阿拉善盟\":\"alsm|10083\",\"海拉尔\":\"hlr|2043\"\n" + + " }\n" + + " ,\n" + + " \"青海\":{\n" + + " \"西宁\":\"xn|2052\",\"海西\":\"hx|9902\",\"海北\":\"haibei|9917\",\"果洛\":\"guoluo|9936\",\"海东\":\"haidong|9909\",\n" + + " \"黄南\":\"huangnan|9896\",\"玉树\":\"ys|9888\",\"海南\":\"hainan|10574\",\"格尔木\":\"geermushi|9904\"\n" + + " }\n" + + " ,\n" + + " \"山东\":{\n" + + " \"青岛\":\"qd|122\",\"济南\":\"jn|265\",\"烟台\":\"yt|228\",\"潍坊\":\"wf|362\",\"临沂\":\"linyi|505\",\n" + + " \"淄博\":\"zb|385\",\"济宁\":\"jining|450\",\"泰安\":\"ta|686\",\"聊城\":\"lc|882\",\"威海\":\"weihai|518\",\n" + + " \"枣庄\":\"zaozhuang|961\",\"德州\":\"dz|728\",\"日照\":\"rizhao|3177\",\"东营\":\"dy|623\",\"菏泽\":\"heze|5632\",\n" + + " \"滨州\":\"bz|944\",\"莱芜\":\"lw|2292\",\"章丘\":\"zhangqiu|8680\",\"垦利\":\"kl|11313\",\"诸城\":\"zc|9146\",\n" + + " \"寿光\":\"shouguang|369\",\"龙口\":\"longkou|233\",\"曹县\":\"caoxian|5638\",\"单县\":\"shanxian|5636\",\"肥城\":\"feicheng|690\",\n" + + " \"高密\":\"gaomi|371\",\"广饶\":\"guangrao|627\",\"桓台\":\"huantaixian|7335\",\"莒县\":\"juxian|3180\",\"莱州\":\"laizhou|235\",\n" + + " \"蓬莱\":\"penglai|237\",\"青州\":\"qingzhou|367\",\"荣成\":\"rongcheng|522\",\"乳山\":\"rushan|520\",\"滕州\":\"tengzhou|967\",\n" + + " \"新泰\":\"xintai|689\",\"招远\":\"zhaoyuan|3325\",\"邹城\":\"zoucheng|455\",\"邹平\":\"zouping|946\",\"临清\":\"linqing|884\",\n" + + " \"茌平\":\"chiping|887\",\"郓城\":\"hzyc|5637\",\"博兴\":\"boxing|949\",\"东明\":\"dongming|5641\",\"巨野\":\"juye|5640\",\n" + + " \"无棣\":\"wudi|951\",\"齐河\":\"qihe|734\",\"微山\":\"weishan|459\",\"禹城\":\"yuchengshi|731\",\"临邑\":\"linyixianq|739\",\n" + + " \"乐陵\":\"leling|730\",\"莱阳\":\"laiyang|234\",\"宁津\":\"ningjin|733\",\"高唐\":\"gaotang|885\",\"莘县\":\"shenxian|888\",\n" + + " \"阳谷\":\"yanggu|886\",\"冠县\":\"guanxian|890\",\"平邑\":\"pingyi|514\",\"郯城\":\"tancheng|510\",\"沂源\":\"yiyuanxian|7334\",\n" + + " \"汶上\":\"wenshang|460\",\"梁山\":\"liangshanx|462\",\"利津\":\"lijin|628\",\"沂南\":\"yinanxian|7301\",\"栖霞\":\"qixia|238\",\n" + + " \"宁阳\":\"ningyang|691\",\"东平\":\"dongping|692\",\"昌邑\":\"changyishi|372\",\"安丘\":\"anqiu|370\",\"昌乐\":\"changle|373\",\n" + + " \"临朐\":\"linqu|374\",\"鄄城\":\"juancheng|5635\"\n" + + " }\n" + + " ,\n" + + " \"山西\":{\n" + + " \"太原\":\"ty|740\",\"临汾\":\"linfen|5669\",\"大同\":\"dt|6964\",\"运城\":\"yuncheng|5653\",\"晋中\":\"jz|8854\",\n" + + " \"长治\":\"changzhi|6921\",\"晋城\":\"jincheng|3350\",\"阳泉\":\"yq|8760\",\"吕梁\":\"lvliang|3222\",\"忻州\":\"xinzhou|3453\",\n" + + " \"朔州\":\"shuozhou|9871\",\"临猗\":\"linyixian|9193\",\"清徐\":\"qingxu|10908\",\"柳林\":\"liulin|3225\",\"高平\":\"gaoping|3354\",\n" + + " \"泽州\":\"zezhou|3353\",\"襄垣\":\"xiangyuanxian|6928\",\"孝义\":\"xiaoyi|3227\"\n" + + " }\n" + + " ,\n" + + " \"陕西\":{\n" + + " \"西安\":\"xa|483\",\"咸阳\":\"xianyang|7453\",\"宝鸡\":\"baoji|2044\",\"渭南\":\"wn|5733\",\"汉中\":\"hanzhong|3163\",\n" + + " \"榆林\":\"yl|5942\",\"延安\":\"yanan|8973\",\"安康\":\"ankang|3157\",\"商洛\":\"sl|9854\",\"铜川\":\"tc|9832\",\n" + + " \"神木\":\"shenmu|5944\",\"韩城\":\"hancheng|5735\",\"府谷\":\"fugu|5945\",\"靖边\":\"jingbian|5947\",\"定边\":\"dingbian|5948\"\n" + + " }\n" + + " ,\n" + + " \"四川\":{\n" + + " \"成都\":\"cd|102\",\"绵阳\":\"mianyang|1057\",\"德阳\":\"deyang|2373\",\"南充\":\"nanchong|2378\",\"宜宾\":\"yb|2380\",\n" + + " \"自贡\":\"zg|6745\",\"乐山\":\"ls|3237\",\"泸州\":\"luzhou|2372\",\"达州\":\"dazhou|9799\",\"内江\":\"scnj|5928\",\n" + + " \"遂宁\":\"suining|9688\",\"攀枝花\":\"panzhihua|2371\",\"眉山\":\"ms|9704\",\"广安\":\"ga|2381\",\"资阳\":\"zy|6803\",\n" + + " \"凉山\":\"liangshan|9717\",\"广元\":\"guangyuan|9749\",\"雅安\":\"ya|9687\",\"巴中\":\"bazhong|9811\",\"阿坝\":\"ab|9817\",\n" + + " \"甘孜\":\"ganzi|9764\",\"安岳\":\"anyuexian|6806\",\"广汉\":\"guanghanshi|8719\",\"简阳\":\"jianyangshi|6805\",\"仁寿\":\"renshouxian|9706\",\n" + + " \"射洪\":\"shehongxian|9694\",\"大竹\":\"dazu|9806\",\"宣汉\":\"xuanhan|9804\",\"渠县\":\"qux|9807\",\"长宁\":\"changningx|7148\"\n" + + " }\n" + + " ,\n" + + " \"新疆\":{\n" + + " \"乌鲁木齐\":\"xj|984\",\"昌吉\":\"changji|8582\",\"巴音郭楞\":\"bygl|9530\",\"伊犁\":\"yili|9472\",\"阿克苏\":\"aks|9499\",\n" + + " \"喀什\":\"ks|9326\",\"哈密\":\"hami|7452\",\"克拉玛依\":\"klmy|2042\",\"博尔塔拉\":\"betl|9529\",\"吐鲁番\":\"tlf|9475\",\n" + + " \"和田\":\"ht|9489\",\"石河子\":\"shz|9551\",\"克孜勒苏\":\"kzls|9519\",\"阿拉尔\":\"ale|9539\",\"五家渠\":\"wjq|9562\",\n" + + " \"图木舒克\":\"tmsk|9559\",\"库尔勒\":\"kel|7168\",\"阿勒泰\":\"alt|18837\",\"塔城\":\"tac|18845\"\n" + + " }\n" + + " ,\n" + + " \"西藏\":{\n" + + " \"拉萨\":\"lasa|2055\",\"日喀则\":\"rkz|9615\",\"山南\":\"sn|9576\",\"林芝\":\"linzhi|9646\",\"昌都\":\"changdu|9648\",\n" + + " \"那曲\":\"nq|9618\",\"阿里\":\"al|9678\",\"日土\":\"rituxian|9682\",\"改则\":\"gaizexian|9684\"\n" + + " }\n" + + " ,\n" + + " \"云南\":{\n" + + " \"昆明\":\"km|541\",\"曲靖\":\"qj|2389\",\"大理\":\"dali|2398\",\"红河\":\"honghe|2394\",\"玉溪\":\"yx|2040\",\n" + + " \"丽江\":\"lj|2392\",\"文山\":\"ws|2395\",\"楚雄\":\"cx|2393\",\"西双版纳\":\"bn|2397\",\"昭通\":\"zt|9409\",\n" + + " \"德宏\":\"dh|9437\",\"普洱\":\"pe|9444\",\"保山\":\"bs|2390\",\"临沧\":\"lincang|9422\",\"迪庆\":\"diqing|9432\",\n" + + " \"怒江\":\"nujiang|9462\",\"弥勒\":\"milexian|8892\",\"安宁\":\"anningshi|547\",\"宣威\":\"xuanwushi|7533\"\n" + + " }\n" + + " ,\n" + + " \"浙江\":{\n" + + " \"杭州\":\"hz|79\",\"宁波\":\"nb|135\",\"温州\":\"wz|330\",\"金华\":\"jh|531\",\"嘉兴\":\"jx|497\",\n" + + " \"台州\":\"tz|403\",\"绍兴\":\"sx|355\",\"湖州\":\"huzhou|831\",\"丽水\":\"lishui|7921\",\"衢州\":\"quzhou|6793\",\n" + + " \"舟山\":\"zhoushan|8481\",\"乐清\":\"yueqingcity|13950\",\"瑞安\":\"ruiancity|13951\",\"义乌\":\"yiwu|12291\",\"余姚\":\"yuyao|5333\",\n" + + " \"诸暨\":\"zhuji|357\",\"象山\":\"xiangshanxian|6738\",\"温岭\":\"wenling|408\",\"桐乡\":\"tongxiang|502\",\"慈溪\":\"cixi|5334\",\n" + + " \"长兴\":\"changxing|834\",\"嘉善\":\"jiashanx|14357\",\"海宁\":\"haining|500\",\"德清\":\"deqing|835\",\"东阳\":\"dongyang|536\",\n" + + " \"安吉\":\"anji|836\",\"苍南\":\"cangnanxian|7579\",\"临海\":\"linhai|407\",\"永康\":\"yongkang|537\",\"玉环\":\"yuhuan|409\",\n" + + " \"平湖\":\"pinghushi|501\",\"海盐\":\"haiyan|504\",\"武义县\":\"wuyix|14528\",\"嵊州\":\"shengzhou|359\",\"新昌\":\"xinchang|361\",\n" + + " \"江山\":\"jiangshanshi|6796\",\"平阳\":\"pingyangxian|7575\"\n" + + " }\n" + + " ,\n" + + " \"其他\":{\n" + + " \"中国香港\":\"hk|2050\",\"中国澳门\":\"am|9399\",\"中国台湾\":\"tw|2051\"\n" + + " }\n" + + " ,\n" + + " \"海外\":{\n" + + " \"洛杉矶\":\"gllosangeles\",\"旧金山\":\"glsanfrancisco\",\"纽约\":\"glnewyork\",\"多伦多\":\"gltoronto\",\"温哥华\":\"glvancouver\",\n" + + " \"伦敦\":\"glgreaterlondon\",\"莫斯科\":\"glmoscow\",\"首尔\":\"glseoul\",\"东京\":\"gltokyo\",\"新加坡\":\"glsingapore\",\n" + + " \"曼谷\":\"glbangkok\",\"清迈\":\"glchiangmai\",\"迪拜\":\"gldubai\",\"奥克兰\":\"glauckland\",\"悉尼\":\"glsydney\",\n" + + " \"墨尔本\":\"glmelbourne\",\"其他海外城市\":\"city\"\n" + + " }\n" + + " }"; + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + JSONObject jsonObject = JSONObject.parseObject(cityList) + Set provinceSet = jsonObject.keySet() + List ret = new ArrayList<>() + for (String province: provinceSet) { + JSONObject provinceInfo = jsonObject[province] + for (String city : provinceInfo.keySet()) { + def city_short = provinceInfo[city].split("\\|")[0] + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", city_short) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "58.com") + ret.add(dbObject) + } + } + cityConn.insert(ret) + print("58城市数据写入成功") + } +} From 66058f0a7152bc5c70f5d4348486c827e46977f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 18:46:34 +0800 Subject: [PATCH 020/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/TaskExport.groovy | 104 +++++------------- .../fangjia/script/statistics/ApiUtil.java | 4 +- .../script/statistics/TaskPersonDTO.java | 31 ++++++ 3 files changed, 61 insertions(+), 78 deletions(-) create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index f2e64dd9e..196e9950b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -3,6 +3,7 @@ import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel import com.fangjia.script.statistics.Task import com.fangjia.script.statistics.TaskDetail +import com.fangjia.script.statistics.TaskPersonDTO import org.apache.commons.collections.CollectionUtils /** @@ -14,89 +15,40 @@ import org.apache.commons.collections.CollectionUtils class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01" - static final String endTime = "2021-04-30" + static final String startTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { - def sortedResult = [] - sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + List list = new ArrayList<>() def result = [] - List projects = ApiUtil.listProject(); - for (Project project : projects) { + result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] + List tasks = ApiUtil.listTaskByPerson(startTime, endTime) + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + + TaskPersonDTO dto = new TaskPersonDTO() + dto.name = task.response.name + dto.project = "" + dto.taskName = task.name + dto.score = Double.valueOf(taskDetail.flowConten) + dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) + + list.add(dto) + } + + Collections.sort(list) + + for (TaskPersonDTO dto : list) { def ret = [] - List labels = ApiUtil.listProjectLabel(project.getId()) - ret[2] = project.getName() - for (ProjectLabel label : labels) { - if ("商务负责人" == label.getGroups().getName()) { - if (ret[3] == null) { - ret[3] = label.getName() - } else { - ret[3] = ret[3] + "、" + label.getName() - } - } else if ("项目类别" == label.getGroups().getName()) { - ret[0] = label.getName() - } - } - // 获取项目下任务 - List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) - Double totalTime = 0D; - if (CollectionUtils.isNotEmpty(tasks)) { - for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - boolean find = false - for (int i = 4; i < sortedResult[0].size(); i++) { - // 找到该项目下该负责人,则加工时 - if (sortedResult[0][i] == taskDetail.getResponse().getName()) { - if (ret[i] == null) { - ret[i] = Double.valueOf(taskDetail.getFlowConten()) - } else { - ret[i] += Double.valueOf(taskDetail.getFlowConten()) - } - find = true - } - } - // 未找到,新增姓名 - if (!find) { - sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() - ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) - } - } - } + ret[0] = dto.name; + ret[1] = dto.project; + ret[2] = dto.taskName; + ret[3] = dto.score; + ret[4] = dto.workTime; result << ret } - def allTotal = [] - for (def label : sort) { - def total = [] - for (def ret : result) { - if (ret[0] == label) { - sortedResult << ret - for (int i = 4; i < ret.size(); i++) { - if (total[i] == null) { - total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i]) - } else { - total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i])) - } - } - } - } - total[1] = 0 - total[2] = "小计" - sortedResult << total - for (int i = 4; i < total.size(); i++) { - if (allTotal[i] == null) { - allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i]) - } else { - allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) - } - total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) - } - } - allTotal[2] = "人员绩效总计" - sortedResult << allTotal - - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计"); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 33c96ae42..10a2f60f4 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -335,8 +335,8 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - //System.out.println(ApiUtil.listTaskByProject("ecd77309ab1c4b83b54ee87b8ddd4f98", "2021-04-01", "2021-04-30")); + System.out.println(ApiUtil.listTaskByProject("d00fcb9c5307499eab8f9798a23f6360", "2021-04-01", "2021-04-30")); //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); - System.out.println(ApiUtil.listTaskByPerson("2021-04-01", "2021-04-30")); + //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java new file mode 100644 index 000000000..d22c4a364 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java @@ -0,0 +1,31 @@ +package com.fangjia.script.statistics; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/10 + */ +@NoArgsConstructor +@Data +public class TaskPersonDTO implements Comparable { + + private String name; + private String project; + private String taskName; + private Double score; + private Double workTime; + + @Override + public int compareTo(TaskPersonDTO o2) { + if (this.getName().equals(o2.getName())) { + if (this.getProject() != null) { + return this.getProject().compareTo(o2.getProject()); + } + } else { + return this.getName().compareTo(o2.getName()); + } + return 0; + } +} From 44351a80214e1c37810722f1cfaa48cb4eda22ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 10:22:53 +0800 Subject: [PATCH 021/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=8D=95=E5=85=83=E6=A0=BC=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 10 ++++- .../com/fangjia/script/util/ExcelUtils.groovy | 37 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 76ea84abe..d08846701 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -18,6 +18,7 @@ class ProjectExport { static final String endTime = "2021-04-30" static void main(String[] args) { + List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] @@ -57,14 +58,21 @@ class ProjectExport { def sortedResult = [] sortedResult << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] for (def label : sort) { + List mergeRowCel = new ArrayList<>() + mergeRowCel.add(sortedResult.size()) for (def ret : result) { if (ret[0] == label) { sortedResult << ret } } + mergeRowCel.add(sortedResult.size() - 1) + mergeRowCel.add(0) + mergeRowCel.add(0) + + mergeData.add(mergeRowCel) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 4cad85c53..203508a51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -1,11 +1,14 @@ package com.fangjia.script.util +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang.StringUtils import org.apache.poi.hssf.record.BoolErrRecord - +import org.apache.poi.hssf.usermodel.HSSFBorderFormatting import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.apache.poi.ss.formula.eval.ErrorEval import org.apache.poi.ss.usermodel.* +import org.apache.poi.ss.util.CellRangeAddress +import org.apache.poi.ss.util.RegionUtil import org.apache.poi.xssf.streaming.SXSSFWorkbook import java.util.concurrent.ExecutorService @@ -235,7 +238,7 @@ public class ExcelUtils{ return super.getProperty(propertyName) } - public static void write(String fileName, List> data, String sheetName){ + public static void write(String fileName, List> data, String sheetName, List> mergeData){ File file = new File(fileName) File dir = file.getParentFile(); @@ -304,6 +307,8 @@ public class ExcelUtils{ throw e } } + mergeCell(workBook, mainSheet, mergeData) + for(int i = 0; i < maxColumnNum; i++ ){ mainSheet.autoSizeColumn(i) } @@ -321,6 +326,34 @@ public class ExcelUtils{ fos.close(); } + static void write(String fileName, List> data, String sheetName){ + write(fileName, data, sheetName, null) + } + + /** + * 合并单元格 + * @param workBook + * @param sheet + * @param data int firstRow, int lastRow, int firstCol, int lastCol + */ + static mergeCell(Workbook workBook, Sheet sheet, List> data) { + if (CollectionUtils.isEmpty(data)) { + return + } + //设置style + CellStyle mergedCellStyle = workBook.createCellStyle() + mergedCellStyle.verticalAlignment = VerticalAlignment.CENTER; + mergedCellStyle.alignment = HorizontalAlignment.CENTER; + + for (List rowCell : data) { + CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3)) + sheet.addMergedRegion(cra); + //设置合并后style + def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2)); + cell.setCellStyle(mergedCellStyle) + } + } + public static newWorkBook(String fileName){ Workbook wb; if(fileName.endsWith(".xlsx")){ From 9c8d0e33a77b69c3953b32260038312ec055765f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 10:42:45 +0800 Subject: [PATCH 022/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=A9=E6=95=88?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=8D=95=E5=85=83=E6=A0=BC=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ScoreExport.groovy | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index baead79bc..710ebd395 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -14,10 +14,11 @@ import org.apache.commons.collections.CollectionUtils class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01" - static final String endTime = "2021-04-30" + static final String startTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { + List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; @@ -40,7 +41,6 @@ class ScoreExport { } // 获取项目下任务 List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) - Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) @@ -68,6 +68,7 @@ class ScoreExport { def allTotal = [] for (def label : sort) { def total = [] + int startRow = sortedResult.size() for (def ret : result) { if (ret[0] == label) { sortedResult << ret @@ -80,8 +81,22 @@ class ScoreExport { } } } + int endRow = sortedResult.size() - 1 + List mergeRowCel = new ArrayList<>() + mergeRowCel.add(startRow) + mergeRowCel.add(endRow) + mergeRowCel.add(0) + mergeRowCel.add(0) + mergeData.add(mergeRowCel) - total[1] = 0 + mergeRowCel = new ArrayList<>() + mergeRowCel.add(startRow) + mergeRowCel.add(endRow) + mergeRowCel.add(1) + mergeRowCel.add(1) + mergeData.add(mergeRowCel) + + double totalNow = 0 total[2] = "小计" sortedResult << total for (int i = 4; i < total.size(); i++) { @@ -90,13 +105,14 @@ class ScoreExport { } else { allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) } - total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) + totalNow += total[i] == null ? 0 : Double.valueOf(total[i]) } + sortedResult[startRow][1] = totalNow } allTotal[2] = "人员绩效总计" sortedResult << allTotal - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计", mergeData); } } From dbcf202d226d754e45801d4306173cfdc2a34a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 11:28:17 +0800 Subject: [PATCH 023/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=B0=8F=E8=AE=A1=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 4 +- .../shenjian/statistics/TaskExport.groovy | 41 ++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index d08846701..317fd8fdb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -14,8 +14,8 @@ import org.apache.commons.collections.CollectionUtils class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01" - static final String endTime = "2021-04-30" + static final String startTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 196e9950b..7d05209bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -19,6 +19,7 @@ class TaskExport { static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { + List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] @@ -38,6 +39,7 @@ class TaskExport { Collections.sort(list) + Map> map = new HashMap<>() for (TaskPersonDTO dto : list) { def ret = [] ret[0] = dto.name; @@ -45,10 +47,47 @@ class TaskExport { ret[2] = dto.taskName; ret[3] = dto.score; ret[4] = dto.workTime; + if (map.containsKey(dto.name)) { + Map info = map.get(dto.name) + info.put("endRow", Double.valueOf(info.get("endRow") + 1)) + info.put("totalScore", Double.valueOf(info.get("totalScore") + dto.score)) + info.put("workTime", Double.valueOf(info.get("workTime") + dto.workTime)) + map.put(dto.name, info); + } else { + if (result.size() > 1) { + def total = [] + total[2] = "小计" + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + result << total + } + Map info = new HashMap<>() + info.put("startRow", Double.valueOf(result.size())) + info.put("endRow", Double.valueOf(result.size())) + info.put("startCell", 0D) + info.put("endCell", 0D) + info.put("totalScore", Double.valueOf(dto.score)) + info.put("workTime", Double.valueOf(dto.workTime)) + map.put(dto.name, info) + } result << ret } + def total = [] + total[2] = "小计" + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + result << total - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计"); + for (Map info : map.values()) { + List data = new ArrayList<>() + data.add(info.get("startRow").toInteger()) + data.add(info.get("endRow").toInteger()) + data.add(info.get("startCell").toInteger()) + data.add(info.get("endCell").toInteger()) + mergeData.add(data) + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计", mergeData); } } From 87c938d283fa613ae79500a9a6312ebfc4662299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 13:39:54 +0800 Subject: [PATCH 024/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 20 ++++++---- .../shenjian/statistics/ScoreExport.groovy | 38 +++++++++++-------- .../shenjian/statistics/TaskExport.groovy | 38 ++++++++++++++----- .../fangjia/script/statistics/ApiUtil.java | 6 +-- 4 files changed, 66 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 317fd8fdb..abff40879 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -15,7 +15,8 @@ class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -43,12 +44,17 @@ class ProjectExport { Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - // 先取绩效,不存在则取工时 - if (taskDetail.getFlowConten() != null) { - totalTime += Double.valueOf(taskDetail.getFlowConten()) - } else { - totalTime += Double.valueOf(taskDetail.getWorkTime()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 先取绩效,不存在则取工时 + if (taskDetail.getFlowConten() != null) { + totalTime += Double.valueOf(taskDetail.getFlowConten()) + } else { + totalTime += Double.valueOf(taskDetail.getWorkTime()) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index 710ebd395..ef71d71de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -15,7 +15,8 @@ class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -43,23 +44,28 @@ class ScoreExport { List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - boolean find = false - for (int i = 4; i < sortedResult[0].size(); i++) { - // 找到该项目下该负责人,则加工时 - if (sortedResult[0][i] == taskDetail.getResponse().getName()) { - if (ret[i] == null) { - ret[i] = Double.valueOf(taskDetail.getFlowConten()) - } else { - ret[i] += Double.valueOf(taskDetail.getFlowConten()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true } - find = true } - } - // 未找到,新增姓名 - if (!find) { - sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() - ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 7d05209bc..dcc5fcccd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -16,25 +16,43 @@ class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] - List tasks = ApiUtil.listTaskByPerson(startTime, endTime) + int limitStart = 0 + List tasks = new ArrayList<>() + List tempTasks = tasks; + + while (tempTasks.isEmpty() || tempTasks.size() == 50) { + tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + limitStart += 50 + tasks.addAll(tempTasks) + } for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } - TaskPersonDTO dto = new TaskPersonDTO() - dto.name = task.response.name - dto.project = "" - dto.taskName = task.name - dto.score = Double.valueOf(taskDetail.flowConten) - dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) + TaskPersonDTO dto = new TaskPersonDTO() + dto.name = task.response.name + dto.project = "" + dto.taskName = task.name + dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten) + dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) - list.add(dto) + list.add(dto) + } } Collections.sort(list) diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 10a2f60f4..a21653dce 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -270,12 +270,12 @@ public class ApiUtil { /** * 鑾峰彇浜哄憳浠诲姟鍒楄〃 */ - public static List listTaskByPerson(String startTime, String endTime) { + public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); - body.put("limitStart", "0"); - body.put("limitNum", "50"); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); //body.put("userid", ""); body.put("type", ""); body.put("timezone", ""); From a8c32d7276c67703b3e29693ca028c9d849b2c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 13:39:54 +0800 Subject: [PATCH 025/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 22 +++++---- .../shenjian/statistics/ScoreExport.groovy | 48 ++++++++++++------- .../shenjian/statistics/TaskExport.groovy | 40 +++++++++++----- .../fangjia/script/statistics/ApiUtil.java | 9 ++-- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 317fd8fdb..f1914cadd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -15,7 +15,8 @@ class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -43,12 +44,17 @@ class ProjectExport { Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - // 先取绩效,不存在则取工时 - if (taskDetail.getFlowConten() != null) { - totalTime += Double.valueOf(taskDetail.getFlowConten()) - } else { - totalTime += Double.valueOf(taskDetail.getWorkTime()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 先取绩效,不存在则取工时 + if (taskDetail.getFlowConten() != null) { + totalTime += Double.valueOf(taskDetail.getFlowConten()) + } else { + totalTime += Double.valueOf(taskDetail.getWorkTime()) + } } } } @@ -72,7 +78,7 @@ class ProjectExport { mergeData.add(mergeRowCel) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index 710ebd395..f5a7287a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -15,7 +15,8 @@ class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -39,27 +40,38 @@ class ScoreExport { ret[0] = label.getName() } } + if (project.getName() == "建行楼盘数据治理项目") { + println('建行楼盘数据治理项目') + } // 获取项目下任务 List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - boolean find = false - for (int i = 4; i < sortedResult[0].size(); i++) { - // 找到该项目下该负责人,则加工时 - if (sortedResult[0][i] == taskDetail.getResponse().getName()) { - if (ret[i] == null) { - ret[i] = Double.valueOf(taskDetail.getFlowConten()) - } else { - ret[i] += Double.valueOf(taskDetail.getFlowConten()) - } - find = true + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + if (taskDetail.getResponse().getName() == '裴银银') { + println('裴银银') + } + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true + } + } + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[sortedResult[0].size() - 1] = Double.valueOf(taskDetail.getFlowConten()) } - } - // 未找到,新增姓名 - if (!find) { - sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() - ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) } } } @@ -112,7 +124,7 @@ class ScoreExport { allTotal[2] = "人员绩效总计" sortedResult << allTotal - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 7d05209bc..182b34019 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -16,25 +16,43 @@ class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] - List tasks = ApiUtil.listTaskByPerson(startTime, endTime) + int limitStart = 0 + List tasks = new ArrayList<>() + List tempTasks = tasks; + + while (tempTasks.isEmpty() || tempTasks.size() == 50) { + tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + limitStart += 50 + tasks.addAll(tempTasks) + } for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } - TaskPersonDTO dto = new TaskPersonDTO() - dto.name = task.response.name - dto.project = "" - dto.taskName = task.name - dto.score = Double.valueOf(taskDetail.flowConten) - dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) + TaskPersonDTO dto = new TaskPersonDTO() + dto.name = task.response.name + dto.project = "" + dto.taskName = task.name + dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten) + dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) - list.add(dto) + list.add(dto) + } } Collections.sort(list) @@ -87,7 +105,7 @@ class TaskExport { mergeData.add(data) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 10a2f60f4..1de09a0c9 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -270,12 +270,12 @@ public class ApiUtil { /** * 鑾峰彇浜哄憳浠诲姟鍒楄〃 */ - public static List listTaskByPerson(String startTime, String endTime) { + public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); - body.put("limitStart", "0"); - body.put("limitNum", "50"); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); //body.put("userid", ""); body.put("type", ""); body.put("timezone", ""); @@ -335,7 +335,8 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - System.out.println(ApiUtil.listTaskByProject("d00fcb9c5307499eab8f9798a23f6360", "2021-04-01", "2021-04-30")); + System.out.println(ApiUtil.listTaskByProject("457f11e89aad42b6a7bb289e8341ac04", "2021-04-01", "2021-05-15")); + System.out.println(ApiUtil.listTaskByProject("41bdf23cbf634035b694f92ba72b067f", "2021-04-01", "2021-05-15")); //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); } From 1e98b447d4daf4f932b5992887272d296a23fe13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 15:48:15 +0800 Subject: [PATCH 026/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A0=E6=A0=87?= =?UTF-8?q?=E7=AD=BE=E9=A1=B9=E7=9B=AE=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 14 +++++++++++-- .../shenjian/statistics/ScoreExport.groovy | 21 ++++++++++++------- .../fangjia/script/statistics/ApiUtil.java | 13 ++++++------ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index f1914cadd..33ffdfee3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -20,7 +20,7 @@ class ProjectExport { static void main(String[] args) { List> mergeData = new ArrayList<>() - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; def result = [] List projects = ApiUtil.listProject(); @@ -40,7 +40,17 @@ class ProjectExport { } } // 获取项目总工时 - List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + int limitStart = 0 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + List tempTasks = tasks; + + while (tasks != null && tempTasks != null && tasks.size() == 50) { + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + limitStart += 50 + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } + } Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index f5a7287a6..ce333aaa7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -22,7 +22,7 @@ class ScoreExport { List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; def result = [] List projects = ApiUtil.listProject(); for (Project project : projects) { @@ -40,11 +40,18 @@ class ScoreExport { ret[0] = label.getName() } } - if (project.getName() == "建行楼盘数据治理项目") { - println('建行楼盘数据治理项目') - } // 获取项目下任务 - List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + int limitStart = 0 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + List tempTasks = tasks; + + while (tasks != null && tempTasks != null && tasks.size() == 50) { + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + limitStart += 50 + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } + } if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { // 获取计划时间在当月的任务 @@ -53,8 +60,8 @@ class ScoreExport { ) { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) boolean find = false - if (taskDetail.getResponse().getName() == '裴银银') { - println('裴银银') + if (taskDetail.getResponse().getName() == '穆良书' && taskDetail.getName() == "模型级联算法方案设计") { + println('穆良书') } for (int i = 4; i < sortedResult[0].size(); i++) { // 找到该项目下该负责人,则加工时 diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 1de09a0c9..40cdb1790 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -236,12 +236,12 @@ public class ApiUtil { * * @return */ - public static List listTaskByProject(String projectId, String startTime, String endTime) { + public static List listTaskByProject(String projectId, String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); - body.put("limitStart", "0"); - body.put("limitNum", "50"); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); body.put("projectId", projectId); body.put("createtimeStart", startTime); body.put("createtimeEnd", endTime); @@ -335,9 +335,10 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - System.out.println(ApiUtil.listTaskByProject("457f11e89aad42b6a7bb289e8341ac04", "2021-04-01", "2021-05-15")); - System.out.println(ApiUtil.listTaskByProject("41bdf23cbf634035b694f92ba72b067f", "2021-04-01", "2021-05-15")); - //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); + ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); + //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); + //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); } } From d1b5c9361e57318fa56fbb236fb2cc9e2111d029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 16:03:45 +0800 Subject: [PATCH 027/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A0=E6=A0=87?= =?UTF-8?q?=E7=AD=BE=E9=A1=B9=E7=9B=AE=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/statistics/DingTalkUtils.groovy | 3 +-- .../temp/shenjian/statistics/ProjectExport.groovy | 10 +++++++++- .../script/temp/shenjian/statistics/ScoreExport.groovy | 8 ++++++-- .../script/temp/shenjian/statistics/TaskExport.groovy | 4 ++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy index f49025b41..19efb9171 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy @@ -1,3 +1,4 @@ + /** * 钉钉接口调用工具类 * @@ -6,8 +7,6 @@ */ class DingTalkUtils { - final String key = "ding0243a5c51f1f4aa8"; - static void main(String[] args) { } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 33ffdfee3..8c8dc6f64 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -19,8 +19,12 @@ class ProjectExport { static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { + export() + } + + public static void export() { List> mergeData = new ArrayList<>() - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] List projects = ApiUtil.listProject(); @@ -59,6 +63,10 @@ class ProjectExport { || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) ) { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } // 先取绩效,不存在则取工时 if (taskDetail.getFlowConten() != null) { totalTime += Double.valueOf(taskDetail.getFlowConten()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index ce333aaa7..dcef469bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -11,7 +11,7 @@ import org.apache.commons.collections.CollectionUtils * @author shenjian * @since 2021/5/10 */ -class ScoreExport { +public class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" @@ -19,10 +19,14 @@ class ScoreExport { static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { + export() + } + + static void export() { List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] List projects = ApiUtil.listProject(); for (Project project : projects) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 182b34019..fc5bf1190 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -20,6 +20,10 @@ class TaskExport { static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { + export() + } + + static void export() { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] From 74f56daa84e580fc0e10db4ff471f8deabbf9728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 17:15:08 +0800 Subject: [PATCH 028/118] =?UTF-8?q?=E6=8C=89=E5=AE=9E=E9=99=85=E5=AE=8C?= =?UTF-8?q?=E6=88=90=E6=97=B6=E9=97=B4=E7=BB=9F=E8=AE=A1=E7=BB=A9=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 30 +++++++---------- .../shenjian/statistics/ScoreExport.groovy | 19 ++++++----- .../shenjian/statistics/TaskExport.groovy | 23 +++++++------ .../fangjia/script/statistics/ApiUtil.java | 33 ++++++++++++++++--- 4 files changed, 61 insertions(+), 44 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 8c8dc6f64..2bd88e38b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -14,15 +14,15 @@ import org.apache.commons.collections.CollectionUtils class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01 00:00:00" - static final String monthEndTime = "2021-04-30 23:59:59" - static final String endTime = "2021-05-15 23:59:59" + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { export() } - public static void export() { + static void export() { List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] @@ -48,9 +48,9 @@ class ProjectExport { List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) List tempTasks = tasks; - while (tasks != null && tempTasks != null && tasks.size() == 50) { - tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { limitStart += 50 + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) if (CollectionUtils.isNotEmpty(tempTasks)) { tasks.addAll(tempTasks) } @@ -59,22 +59,16 @@ class ProjectExport { if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { // 获取计划时间在当月的任务 - if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) - || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) - ) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - if (taskDetail == null) { - print("当前任务taskDetail为空:" + task) - continue; - } - // 先取绩效,不存在则取工时 + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) if (taskDetail.getFlowConten() != null) { totalTime += Double.valueOf(taskDetail.getFlowConten()) - } else { - totalTime += Double.valueOf(taskDetail.getWorkTime()) } } - } } ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, totalTime) result << ret diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index dcef469bd..81b26e748 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -14,9 +14,9 @@ import org.apache.commons.collections.CollectionUtils public class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01 00:00:00" - static final String monthEndTime = "2021-04-30 23:59:59" - static final String endTime = "2021-05-15 23:59:59" + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { export() @@ -49,9 +49,9 @@ public class ScoreExport { List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) List tempTasks = tasks; - while (tasks != null && tempTasks != null && tasks.size() == 50) { - tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { limitStart += 50 + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) if (CollectionUtils.isNotEmpty(tempTasks)) { tasks.addAll(tempTasks) } @@ -59,10 +59,11 @@ public class ScoreExport { if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { // 获取计划时间在当月的任务 - if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) - || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) - ) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + continue; + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { boolean find = false if (taskDetail.getResponse().getName() == '穆良书' && taskDetail.getName() == "模型级联算法方案设计") { println('穆良书') diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index fc5bf1190..57e76d265 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -15,9 +15,9 @@ import org.apache.commons.collections.CollectionUtils class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01 00:00:00" - static final String monthEndTime = "2021-04-30 23:59:59" - static final String endTime = "2021-05-15 23:59:59" + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { export() @@ -39,15 +39,14 @@ class TaskExport { } for (Task task: tasks) { // 获取计划时间在当月的任务 - if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) - || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) - ) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - if (taskDetail == null) { - print("当前任务taskDetail为空:" + task) - continue; - } - + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + continue; + } + if ("穆良书" == task.getResponse().getName() && task.getId() == "e3f81b0381b44696b06fb6242a44902f") { + print("穆良书") + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { TaskPersonDTO dto = new TaskPersonDTO() dto.name = task.response.name dto.project = "" diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 40cdb1790..cf3128b61 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -18,6 +18,7 @@ import javax.crypto.spec.SecretKeySpec; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; @@ -270,13 +271,13 @@ public class ApiUtil { /** * 鑾峰彇浜哄憳浠诲姟鍒楄〃 */ - public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { + public static List listTaskByPerson(String userid, String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); body.put("limitStart", limitStart); body.put("limitNum", limitNum); - //body.put("userid", ""); + body.put("userid", userid); body.put("type", ""); body.put("timezone", ""); body.put("createtimeStart", startTime); @@ -303,6 +304,12 @@ public class ApiUtil { return tasks; } + /** + * 鑾峰彇浜哄憳浠诲姟鍒楄〃 + */ + public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { + return listTaskByPerson(null, startTime, endTime, limitStart, limitNum); + } /** * 鑾峰彇浠诲姟璇︽儏 * @@ -335,10 +342,26 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); - ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); - //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); + //System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); + //ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); + System.out.println(ApiUtil.getTaskDetail("e3f81b0381b44696b06fb6242a44902f")); //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); +// int limitStart = 0; +// List tasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50); +// List tempTasks = tasks; +// +// while (tasks != null && tempTasks != null && tempTasks.size() == 50) { +// limitStart += 50; +// tempTasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50); +// if (CollectionUtils.isNotEmpty(tempTasks)) { +// tasks.addAll(tempTasks); +// } +// } +// for (Task task : tasks) { +// if ("绌嗚壇涔�".equals(task.getResponse().getName())) { +// System.out.println(task); +// } +// } } } From eca628115f9ac473baaaa694131cb0f17a3c2a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 17:48:47 +0800 Subject: [PATCH 029/118] =?UTF-8?q?=E8=B0=83=E7=94=A8=E8=B6=85=E9=99=90?= =?UTF-8?q?=E5=90=8E=E7=9D=A1=E7=9C=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 5 ++++- .../shenjian/statistics/ScoreExport.groovy | 4 +++- .../shenjian/statistics/TaskExport.groovy | 4 +++- .../fangjia/script/statistics/ApiUtil.java | 20 ++++++++++++++----- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 2bd88e38b..0a243b841 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -22,6 +22,8 @@ class ProjectExport { export() } + + static void export() { List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; @@ -62,7 +64,8 @@ class ProjectExport { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) if (taskDetail == null) { print("当前任务taskDetail为空:" + task) - continue; + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) if (taskDetail.getFlowConten() != null) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index 81b26e748..58285190c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -61,7 +61,9 @@ public class ScoreExport { // 获取计划时间在当月的任务 TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) if (taskDetail == null) { - continue; + print("当前任务taskDetail为空:" + task) + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { boolean find = false diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 57e76d265..664b98e00 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -41,7 +41,9 @@ class TaskExport { // 获取计划时间在当月的任务 TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) if (taskDetail == null) { - continue; + print("当前任务taskDetail为空:" + task) + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) } if ("穆良书" == task.getResponse().getName() && task.getId() == "e3f81b0381b44696b06fb6242a44902f") { print("穆良书") diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index cf3128b61..977bc2888 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -292,12 +292,17 @@ public class ApiUtil { param.put("corpid", corpId); param.put("body", body); JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByPerson", param); - if ((Integer)result.get("code") == 0) { + int code = (Integer)result.get("code"); + if (code == 0) { if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); tasks = jsonArray.toJavaList(Task.class); } - } + } else if (code == 90001){ + throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:listTaskByPerson"); + } else if (code == 90009) { + throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:listTaskByPerson"); + } } catch (Exception ex) { ex.printStackTrace(); } @@ -330,10 +335,15 @@ public class ApiUtil { param.put("corpid", corpId); param.put("body", body); JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskDetail", param); - if ((Integer)result.get("code") == 0) { + int code = (Integer)result.get("code"); + if (code == 0) { JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data"))); detail = obj.toJavaObject(TaskDetail.class); - } + } else if (code == 90001){ + throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:getTaskDetail"); + } else if (code == 90009) { + throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:getTaskDetail"); + } } catch (Exception e) { e.printStackTrace(); } @@ -344,7 +354,7 @@ public class ApiUtil { //List projects = ApiUtil.listProject(); //System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); //ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); - System.out.println(ApiUtil.getTaskDetail("e3f81b0381b44696b06fb6242a44902f")); + System.out.println(ApiUtil.getTaskDetail("be85282dc57d46e3a36fc0801107e605")); //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); // int limitStart = 0; From 14de7390895b098f60d27e45f57037f5e232effb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 10:28:34 +0800 Subject: [PATCH 030/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=A9=E6=95=88?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/dingding/DingTalkUtils.groovy | 24 +++++++++++++++++++ .../ProjectExport.groovy | 10 ++++---- .../ScoreExport.groovy | 8 ++++--- .../TaskExport.groovy | 20 +++++++++------- .../shenjian/statistics/DingTalkUtils.groovy | 13 ---------- 5 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{statistics => dingding}/ProjectExport.groovy (95%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{statistics => dingding}/ScoreExport.groovy (96%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{statistics => dingding}/TaskExport.groovy (90%) delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy new file mode 100644 index 000000000..5dda32688 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -0,0 +1,24 @@ +package com.fangjia.script.temp.shenjian.dingding + +import com.fangjia.script.util.DateUtils + + +/** + * 钉钉接口调用工具类 + * + * @author shenjian + * @since 2021/5/8 + */ +class DingTalkUtils { + + static main(String[] args) { + Date now = DateUtils.parseFromTightByPattern("2021-04-01", "yyyy-MM-dd") ; + + String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" + String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") + String endTime = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(now), "yyyy-MM-dd") + " 23:59:59" + ScoreExport.export(startTime, monthStartTime, endTime) + ProjectExport.export(monthStartTime, monthStartTime, endTime) + TaskExport.export(endTime, monthStartTime, endTime) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy similarity index 95% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy index 0a243b841..efd9a2e3b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy @@ -1,3 +1,5 @@ +package com.fangjia.script.temp.shenjian.dingding + import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel @@ -18,13 +20,11 @@ class ProjectExport { static final String monthStartTime = "2021-04-01 00:00:00" static final String endTime = "2021-04-30 23:59:59" - static void main(String[] args) { - export() + static main(args) { + export(startTime, monthStartTime, endTime) } - - - static void export() { + static export(def startTime, def monthStartTime, def endTime) { List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy similarity index 96% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy index 58285190c..511c72846 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy @@ -1,3 +1,5 @@ +package com.fangjia.script.temp.shenjian.dingding + import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel @@ -18,11 +20,11 @@ public class ScoreExport { static final String monthStartTime = "2021-04-01 00:00:00" static final String endTime = "2021-04-30 23:59:59" - static void main(String[] args) { - export() + static main(args) { + export(startTime, monthStartTime, endTime) } - static void export() { + static void export(def startTime, def monthStartTime, def endTime) { List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy similarity index 90% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 664b98e00..632d6ecb9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -1,3 +1,5 @@ +package com.fangjia.script.temp.shenjian.dingding + import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel @@ -12,30 +14,32 @@ import org.apache.commons.collections.CollectionUtils * @author shenjian * @since 2021/5/10 */ -class TaskExport { +public class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-03-01 00:00:00" static final String monthStartTime = "2021-04-01 00:00:00" static final String endTime = "2021-04-30 23:59:59" - static void main(String[] args) { - export() + static main(args) { + export(startTime, monthStartTime, endTime) } - static void export() { + static void export(def startTime, def monthStartTime, def endTime) { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] int limitStart = 0 - List tasks = new ArrayList<>() + List tasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) List tempTasks = tasks; - while (tempTasks.isEmpty() || tempTasks.size() == 50) { - tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { limitStart += 50 - tasks.addAll(tempTasks) + tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } } for (Task task: tasks) { // 获取计划时间在当月的任务 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy deleted file mode 100644 index 19efb9171..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy +++ /dev/null @@ -1,13 +0,0 @@ - -/** - * 钉钉接口调用工具类 - * - * @author shenjian - * @since 2021/5/8 - */ -class DingTalkUtils { - - static void main(String[] args) { - - } -} \ No newline at end of file From fed7c8763f270582f5872b003579ff40a9a1a69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 10:56:56 +0800 Subject: [PATCH 031/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=8F=E8=AE=A1?= =?UTF-8?q?=E5=AF=BC=E5=87=BAbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/dingding/TaskExport.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 632d6ecb9..a86e1a959 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -101,8 +101,14 @@ public class TaskExport { } def total = [] total[2] = "小计" - total[3] = map.get(result[result.size() - 1][0]).get("totalScore") - total[4] = map.get(result[result.size() - 1][0]).get("workTime") + if (map.get(result[result.size() - 1][0]) != null) { + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + } else { + total[3] = 0 + total[4] = 0 + print("无该信息" + result[result.size() - 1][0]) + } result << total for (Map info : map.values()) { From e20f9c22f1aad5c996753994ea40ea109a848ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 10:56:56 +0800 Subject: [PATCH 032/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BC=A0=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/dingding/DingTalkUtils.groovy | 4 ++-- .../script/temp/shenjian/dingding/TaskExport.groovy | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 5dda32688..00e7644b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -18,7 +18,7 @@ class DingTalkUtils { String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") String endTime = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(now), "yyyy-MM-dd") + " 23:59:59" ScoreExport.export(startTime, monthStartTime, endTime) - ProjectExport.export(monthStartTime, monthStartTime, endTime) - TaskExport.export(endTime, monthStartTime, endTime) + ProjectExport.export(startTime, monthStartTime, endTime) + TaskExport.export(startTime, monthStartTime, endTime) } } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 632d6ecb9..a86e1a959 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -101,8 +101,14 @@ public class TaskExport { } def total = [] total[2] = "小计" - total[3] = map.get(result[result.size() - 1][0]).get("totalScore") - total[4] = map.get(result[result.size() - 1][0]).get("workTime") + if (map.get(result[result.size() - 1][0]) != null) { + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + } else { + total[3] = 0 + total[4] = 0 + print("无该信息" + result[result.size() - 1][0]) + } result << total for (Map info : map.values()) { From 07e725fe6e4cafe792b20964368763b3109aa36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 14:32:09 +0800 Subject: [PATCH 033/118] =?UTF-8?q?58=E7=A7=8D=E5=AD=90=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/crawl/City58.groovy | 26 +++++++---- .../script/temp/shenjian/crawl/Seed58.groovy | 44 +++++++++++++++++++ .../script/temp/shenjian/crawl/SeedBak.groovy | 33 ++++++++++++++ 3 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy index 78f033fd4..8b9abf395 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy @@ -252,23 +252,31 @@ class City58 { " }"; static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) static void main(String[] args) { JSONObject jsonObject = JSONObject.parseObject(cityList) Set provinceSet = jsonObject.keySet() List ret = new ArrayList<>() + + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + for (String province: provinceSet) { JSONObject provinceInfo = jsonObject[province] for (String city : provinceInfo.keySet()) { - def city_short = provinceInfo[city].split("\\|")[0] - DBObject dbObject = new BasicDBObject() - dbObject.put("province", province) - dbObject.put("city", city) - dbObject.put("city_short", city_short) - dbObject.put("c_date", new Date()) - dbObject.put("m_date", new Date()) - dbObject.put("root", "58.com") - ret.add(dbObject) + // 屏蔽县 + if (citySet.contains(city)) { + def city_short = provinceInfo[city].split("\\|")[0] + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", city_short) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "58.com") + ret.add(dbObject) + } } } cityConn.insert(ret) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy new file mode 100644 index 000000000..79bf00e9a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 58有效种子梳理 + * + * @author shenjian + * @since 2021/5/14 + */ +class Seaweed58 { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + + DBObject query = new BasicDBObject() + query.put("root", "58.com") + + DBObject filter = new BasicDBObject() + filter.put("city", "1") + filter.put("_id", "1") + def delIds = [] + districtSeedsConn.find(query, filter).batchSize(20000).each {obj -> + if (!citySet.contains(obj.city)) { + delIds.add(obj._id) + if (delIds.size() == 10000) { + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + } + } + } + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + print("58种子清理成功") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy new file mode 100644 index 000000000..56a389f28 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 种子数据备份 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeedBak { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection aimConn = MongoUtils.getCollectionByName("seeddb_dev", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + List list = new ArrayList<>() + + districtSeedsConn.find().batchSize(20000).each {obj -> + list.add(obj) + if (list.size() == 20000) { + aimConn.insert(list) + list.clear() + } + } + aimConn.insert(list) + list.clear() + println("种子备份成功") + } +} From 07f1398a92d7194c8c54f00f7461019094988d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 17:42:30 +0800 Subject: [PATCH 034/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E7=89=A9=E4=B8=9A=E7=AD=89?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy new file mode 100644 index 000000000..16e797245 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 重庆农商行 + * 开发商物业公司等小区信息导出 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeaweedExport { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static def map = new ConcurrentHashMap() + + static void main(String[] args) { + def result = new ConcurrentLinkedQueue() + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", name) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("developer", 1) + filter.put("estate_company", 1) + filter.put("cubage_percent", 1) // 容积率 + filter.put("virescence_percent", 1) // 绿化率 + filter.put("alias", 1) + filter.put("car_site_count", 1) + filter.put("all_area", 1) + filter.put("loudong_count", 1) + filter.put("household_count", 1) + filter.put("estate_charge", 1) + filter.put("open_date", 1) // 开盘时间 + filter.put("live_date", 1) // 交房时间 + + DbUtils.seaweed.find(query).batchSize(10).each { obj -> + def ret = [] + ret[0] = obj.developer + ret[1] = obj.estate_company + ret[2] = obj.cubage_percent + ret[3] = obj.virescence_percent + + if (obj.alias) { + // 小区可能存在多个别名 + def temp = [] + for (def a : obj.alias) { + temp.add(a) + } + ret[4] = temp.join(",") + } + ret[5] = obj.car_site_count + ret[6] = "" + ret[7] = "" + ret[8] = obj.all_area + ret[9] = obj.loudong_count + ret[10] = obj.household_count + ret[11] = "" + ret[12] = obj.estate_charge + ret[13] = obj.open_date + ret[14] = obj.live_date + result.add(ret) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret = [] + for (def r : result) { + ret << r + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商物业公司.xlsx", ret, "项目统计"); + executorService.shutdown() + } +} From 30d5583a52b1ca74ad2d4be21d677d36874ed17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 17:42:30 +0800 Subject: [PATCH 035/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E7=89=A9=E4=B8=9A=E7=AD=89?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AroundExport.groovy | 18 +++ .../temp/shenjian/cqrcb/Constants.groovy | 10 ++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 126 ++++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 60 +++++++++ 4 files changed, 214 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy new file mode 100644 index 000000000..40851320a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -0,0 +1,18 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + +/** + * 小区周围信息导出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AroundExport { + + def static types = "" + + static void main(String[] args) { + GaoDeUtils.searchNearByPoi() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy new file mode 100644 index 000000000..bfafad05d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy @@ -0,0 +1,10 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +/** + * @author shenjian + * @since 2021/5/15 + */ +class Constants { + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy new file mode 100644 index 000000000..445b953f6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -0,0 +1,126 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 重庆农商行 + * 开发商物业公司等小区信息导出 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeaweedExport { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static def map = new ConcurrentHashMap() + + static void main(String[] args) { + def resultMap = new ConcurrentHashMap() + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", name) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("developer", 1) + filter.put("estate_company", 1) + filter.put("cubage_percent", 1) // 容积率 + filter.put("virescence_percent", 1) // 绿化率 + filter.put("alias", 1) + filter.put("car_site_count", 1) + filter.put("all_area", 1) + filter.put("loudong_count", 1) + filter.put("household_count", 1) + filter.put("estate_charge", 1) + filter.put("open_date", 1) // 开盘时间 + filter.put("live_date", 1) // 交房时间 + filter.put("lng", 1) + filter.put("lat", 1) + + def ret = [] + DbUtils.seaweed.find(query).limit(1).each { obj -> + ret[0] = obj.developer + ret[1] = obj.estate_company + ret[2] = obj.cubage_percent + ret[3] = obj.virescence_percent + + if (obj.alias) { + // 小区可能存在多个别名 + def temp = [] + for (def a : obj.alias) { + temp.add(a) + } + ret[4] = temp.join(",") + } + ret[5] = obj.car_site_count + ret[6] = "" + ret[7] = "" + ret[8] = obj.all_area + ret[9] = obj.loudong_count + ret[10] = obj.household_count + ret[11] = "" + ret[12] = obj.estate_charge + ret[13] = obj.open_date + ret[14] = obj.live_date + ret[15] = obj.lng + ret[16] = obj.lat + } + ret[17] = name + resultMap.put(district_code + name, ret) + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret = [] + ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"] + for (List district : data) { + def district_code = district.get(2) + def name = district.get(3) + ret << resultMap.get(district_code + name) + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商物业公司.xlsx", ret, "项目统计"); + executorService.shutdown() + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..155740221 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,60 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.HttpRequestUtil +import net.sf.json.JSONObject + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde" + + /** + * + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, String radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = radius + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else { + break + } + } + return result + } +} From 0793de69327edd760ff6bd5d8b95c7fca18d3c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 15 May 2021 11:52:30 +0800 Subject: [PATCH 036/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=B0=8F=E5=8C=BA=E5=91=A8=E5=9B=B4=E5=BB=BA=E7=AD=91?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AroundExport.groovy | 50 ++++++++++++++++++- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 1 - 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy index 40851320a..9c53f103f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -1,6 +1,18 @@ package com.fangjia.script.temp.shenjian.cqrcb +import cn.emay.mina.util.ConcurrentHashSet +import com.fangjia.script.coredata.util.CollectionUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger /** * 小区周围信息导出 @@ -10,9 +22,43 @@ import com.fangjia.script.temp.shenjian.util.GaoDeUtils */ class AroundExport { - def static types = "" + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - GaoDeUtils.searchNearByPoi() + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + def city = district.get(1) + def name = district.get(3) + if (district.get(20) != null && district.get(21) != null) { + boolean exist = conn.count(new BasicDBObject("district_name", name)) > 0 ? true : false + if (!exist) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("district_name", name) + obj.put("lng", lng) + obj.put("lat", lat) + } + conn.insert(arounds) + } + } + } + println("当前处理小区" + name + " " + count.incrementAndGet()) + } + }) + } + executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 445b953f6..8d48e0319 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -8,7 +8,6 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors From 6a679dc5fcd6f6db6adb70b1acf34c21a147b0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 15 May 2021 14:04:53 +0800 Subject: [PATCH 037/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=B0=8F=E5=8C=BA=E5=91=A8=E5=9B=B4=E5=BB=BA=E7=AD=91?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AllExport.groovy | 74 ++++++++ .../temp/shenjian/cqrcb/AroundExport.groovy | 55 ++++-- .../cqrcb/AroundExportByNewWay.groovy | 174 ++++++++++++++++++ .../shenjian/cqrcb/AroundFormatExport.groovy | 161 ++++++++++++++++ .../shenjian/cqrcb/MarkUnNeedExport.groovy | 32 ++++ .../shenjian/cqrcb/MoveArroundTemp.groovy | 64 +++++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 47 ++++- .../temp/shenjian/school/DistanceDeal.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 3 +- 9 files changed, 583 insertions(+), 31 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy new file mode 100644 index 000000000..99bf24d62 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy @@ -0,0 +1,74 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 重庆所有小区周边导出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AllExport { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + AtomicInteger count = new AtomicInteger(0) + DBUtils.seaweed.find(query, filter).batchSize(10000).each {district -> + executorService.execute(new Runnable() { + @Override + void run() { + def city = "重庆" + def name = district.name + def region = district.region + boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false + if (!exist) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lng / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lat / 10000000000d) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("matched_city", city) + obj.put("matched_region", district.region) + obj.put("matched_name", district.name) + } + conn.insert(arounds) + } + println("当前处理小区" + name + " " + count.incrementAndGet()) + } else { + println("小区已存在") + } + } + }) + } + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy index 9c53f103f..fd3450f87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.temp.shenjian.cqrcb import cn.emay.mina.util.ConcurrentHashSet import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoUtils @@ -27,38 +28,54 @@ class AroundExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" static def radius = "3000" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + //static ExecutorService executorService = Executors.newFixedThreadPool(30) + // 高德地图中不存在周边的数据 + // 重庆 江北 阳光丽苑 + //重庆 江北 金田花苑 + //重庆 巴南 巴滨一号璀璨滨江 + //重庆 丰都 茅林沟移民新村 + //重庆 江津 福城北苑 + //重庆 巫山 摩天岭壹号 static void main(String[] args) { - List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 0) AtomicInteger count = new AtomicInteger(0) for (List districtT : data) { final List district = districtT - executorService.execute(new Runnable() { - @Override - void run() { - def city = district.get(1) - def name = district.get(3) - if (district.get(20) != null && district.get(21) != null) { - boolean exist = conn.count(new BasicDBObject("district_name", name)) > 0 ? true : false - if (!exist) { - def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) - def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) +// executorService.execute(new Runnable() { +// @Override +// void run() { + def city = district.get(0) + def region = district.get(1) + def name = district.get(2) + boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false + if (!exist) { + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("region", region) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", name) + DBUtils.seaweed.find(query, new BasicDBObject("lat", 1).append("lng", 1)).limit(1).each {objT -> + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lng / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lat / 10000000000d) List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") if (CollectionUtils.isNotEmpty(arounds)) { for (BasicDBObject obj : arounds) { - obj.put("district_name", name) - obj.put("lng", lng) - obj.put("lat", lat) + obj.put("matched_city", city) + obj.put("matched_region", region) + obj.put("matched_name", name) } conn.insert(arounds) } + println("当前处理小区" + name + " " + count.incrementAndGet()) } + } else { + println("小区已存在") } - println("当前处理小区" + name + " " + count.incrementAndGet()) - } - }) +// } +// }) } - executorService.shutdown() + //executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy new file mode 100644 index 000000000..7053f1db6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -0,0 +1,174 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 周边数据导出新方式 + * + * @author shenjian + * @since 2021/5/16 + */ +class AroundExportByNewWay { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + static def resultMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def id = district.get(0) + def district_code = district.get(2) + if (district_code == 'B100203') { + println(district_code) + } + def name = district.get(3) + List result = [] + if (district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) + DBObject q = new BasicDBObject() + q.put('status', 1) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + Map map = new ConcurrentHashMap<>(); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 3000) { + dealSpecial(obj, map, id, distance, result) + } + } + for (String key : map.keySet()) { + int distance = map.get(key).get("distance") + int count1 = map.get(key).get("count") + def type = map.get(key).get("type") + def ret = [] + ret[0] = id + ret[1] = key + ret[2] = type + ret[3] = (distance / count1).toInteger() + result << ret + } + } + resultMap.put(district_code + name, result) + println("当前处理" + count.incrementAndGet() ) + } catch (Exception ex) { + + } finally { + latch.countDown() + } + } + }) + }; + latch.await() + def ret = [] + ret << ['IDB', '名称','类型', '距离'] + for (List district : data) { + def district_code = district.get(2) + def name = district.get(3) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + ret.addAll(resultMap.get(district_code + name)) + } + } + println(ret.size()) + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + executorService.shutdown() + + } + + static dealSpecial(DBObject obj, def map, def id, double distance, def result) { + def name = obj.name + if (name.indexOf("/") != -1) { + return false + } + if (name == "1号楼") { + return false + } + if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { + if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { + if (name.indexOf("分院") == -1) { + return false + } + } + if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { + return false + } + if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { + if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 + && !name.endsWith("医院")) { + return false + } + } + if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { + return false + } + if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { + return false + } + if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { + if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { + return false + } + } + if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + } + // 中梁山地铁站1A口 + if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { + def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) + def key = nameTemp + Map child = new HashMap() + if (map.get(key) != null) { + distance + map.get(key).get("distance") + int count1 = map.get(key).get("count") + child.put("distance", distance) + child.put("count", count1) + child.put("type", obj.type) + map.put(key, child) + } else { + child.put("distance", distance) + child.put("type", obj.type) + child.put("count", 1) + map.put(key, child) + } + } else { + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = obj.type + ret[3] = distance.toInteger() + result << ret + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy new file mode 100644 index 000000000..b445e881c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -0,0 +1,161 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import cn.emay.mina.util.ConcurrentHashSet +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区周围信息格式化输出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AroundFormatExport { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static def resultMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def district_code = district.get(2) + def name = district.get(3) + def temp = [] + if (district.get(20) != null && district.get(21) != null) { + def matched_city = district.get(22) + def matched_region = district.get(23) + def matched_name = district.get(24) + + DBObject query = new BasicDBObject() + query.put("matched_city", matched_city) + query.put("matched_region", matched_region) + query.put("matched_name", matched_name) + List ret = new ArrayList() + Map map = new ConcurrentHashMap<>(); + conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> + dealSpecial(obj, map, ret) + } + for (String key : map.keySet()) { + int distance = map.get(key).get("distance") + int count1 = map.get(key).get("count") + ret.add(key + ":" + (distance / count1) + "米") + } + + temp[7] = matched_city + temp[8] = matched_region + temp[9] = matched_name + if (ret.size() > 0) { + def str = ret.join(",") + temp[5] = str + println(str) + } + + } + resultMap.put(district_code + name, temp) + println("当前处理" + count.incrementAndGet() ) + } catch (Exception ex) { + + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] + for (List district : data) { + def district_code = district.get(2) + def name = district.get(3) + ret << resultMap.get(district_code + name) + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + executorService.shutdown() + } + + static dealSpecial(DBObject obj, def map, def ret) { + def name = obj.name + if (name.indexOf("/") != -1) { + return false + } + if (name == "1号楼") { + return false + } + if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { + if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { + if (name.indexOf("分院") == -1) { + return false + } + } + if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { + return false + } + if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { + if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 + && !name.endsWith("医院")) { + return false + } + } + if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { + return false + } + if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { + return false + } + if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { + if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { + return false + } + } + if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + } + // 中梁山地铁站1A口 + if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { + def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) + def key = nameTemp + Map child = new HashMap() + if (map.get(key) != null) { + int distance = Double.valueOf(obj.distance) + map.get(key).get("distance") + int count1 = map.get(key).get("count") + child.put("distance", distance) + child.put("count", count1) + map.put(key, child) + } else { + child.put("distance", Double.valueOf(obj.distance)) + child.put("count", 1) + map.put(key, child) + } + } else { + ret.add(obj.name + ":" + obj.distance + "米") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy new file mode 100644 index 000000000..76226f770 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +/** + * 标记不需要导出的数据 + * + * @author shenjian + * @since 2021/5/16 + */ +class MarkUnNeedExport { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 1) + for (List districtT : data) { + final List district = districtT + if (district.size() > 1) { + def type = district.get(0) + def status = district.get(1) + if (status == 1) { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1))) + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy new file mode 100644 index 000000000..e17e12840 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * POI迁移 + * + * @author shenjian + * @since 2021/5/16 + */ +class MoveArroundTemp { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List list = Collections.synchronizedList(new ArrayList()) + List ids = conn.distinct("id") + CountDownLatch latch = new CountDownLatch(ids.size()) + for (String idT : ids) { + final String id = idT + executorService.submit(new Runnable() { + @Override + void run() { + try { + DBObject dbObject = new BasicDBObject() + conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> + dbObject.put("_id", id) + dbObject.put("name", obj.name) + dbObject.put("type", obj.type) + dbObject.put("typecode", obj.typecode) + dbObject.put("address", obj.address) + dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) + dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) + dbObject.put("tel", obj.tel) + dbObject.put("province", obj.pname) + dbObject.put("city", obj.cityname) + dbObject.put("region", obj.adname) + } + list.add(dbObject) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + println("当前记录数" + list.size()) + } + } + }) + }; + latch.await(); + conn43.insert(list); + list.clear(); + println("同步清理完成"); + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 8d48e0319..feb17ea7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.cqrcb +import com.fangjia.core.util.DateUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -38,7 +39,11 @@ class SeaweedExport { def city = district.get(1) def district_code = district.get(2) def name = district.get(3) + def matchedCity = district.get(22) + def matchedRegion = district.get(23) + def matchedName = district.get(24) def region; + if (map.get("district_code") != null) { region = map.get("district_code") } else { @@ -47,12 +52,17 @@ class SeaweedExport { map.put("district_code", region) } } - + if (matchedRegion == null) { + matchedRegion = region + matchedName = name + } + def ret = [] DBObject query = new BasicDBObject() query.put("city", city) + query.put("region", matchedRegion) query.put("cat", "district") query.put("visible", 0) - query.put("name", name) + query.put("name", matchedName) DBObject filter = new BasicDBObject() filter.put("_id", 1) @@ -71,12 +81,11 @@ class SeaweedExport { filter.put("lng", 1) filter.put("lat", 1) - def ret = [] DbUtils.seaweed.find(query).limit(1).each { obj -> ret[0] = obj.developer ret[1] = obj.estate_company ret[2] = obj.cubage_percent - ret[3] = obj.virescence_percent + ret[3] = virescencePercentValidator(obj.virescence_percent) if (obj.alias) { // 小区可能存在多个别名 @@ -94,7 +103,7 @@ class SeaweedExport { ret[10] = obj.household_count ret[11] = "" ret[12] = obj.estate_charge - ret[13] = obj.open_date + ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date) ret[14] = obj.live_date ret[15] = obj.lng ret[16] = obj.lat @@ -108,9 +117,9 @@ class SeaweedExport { } } }) - } - latch.await() - def ret = [] + }; + latch.await(); + def ret = []; ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"] for (List district : data) { def district_code = district.get(2) @@ -121,5 +130,27 @@ class SeaweedExport { executorService.shutdown() } + /** + * string + * 绿化率 仅限数字,包括(小数)数字范围:10-100 + * @param virescence_percent + * @return + */ + private static def virescencePercentValidator(def virescence_percent){ + if(virescence_percent=="null"){ + return ""; + } + def value = com.fangjia.script.datastore.DistrictFieldNumberic.checkValue(virescence_percent); + if(value){ + if(value>=10 && value<=100){ + return value; + } else if (value > 0 && value < 1) { + return value * 100 + } else{ + return ""; + } + } + return ""; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy index d19524f07..3037ee807 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -33,8 +33,8 @@ class DistanceDeal { // updateField.put("school_lat", "31.330963") // collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true) -// double length = Distance.GetDistance(); -// print(length) + double length = Distance.GetDistance(); + print(length) // DBObject updateField = new BasicDBObject() // updateField.put("school_lng", 120.68764) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 155740221..eef249210 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,7 +1,6 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.core.util.DataUtil -import com.fangjia.script.util.GeoUtils import com.fangjia.script.util.HttpRequestUtil import net.sf.json.JSONObject @@ -13,7 +12,7 @@ import net.sf.json.JSONObject */ class GaoDeUtils { - def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde" + def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988" /** * From 37580693794fb056fae5ea093e94bcd537ae5def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 16 May 2021 16:06:51 +0800 Subject: [PATCH 038/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=B0=8F=E5=8C=BA=E5=91=A8=E5=9B=B4=E5=BB=BA=E7=AD=91?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 22 +++++++++++++++---- .../shenjian/cqrcb/MarkUnNeedExport.groovy | 11 ++++++++-- .../temp/shenjian/util/GaoDeUtils.groovy | 16 ++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 7053f1db6..94e4b9342 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -1,13 +1,16 @@ package com.fangjia.script.temp.shenjian.cqrcb +import com.alibaba.excel.EasyExcel import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -27,6 +30,12 @@ class AroundExportByNewWay { static def resultMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) + static Map map = new HashedMap() + + static { + map.put("") + } + static void main(String[] args) { List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) AtomicInteger count = new AtomicInteger(0) @@ -51,11 +60,11 @@ class AroundExportByNewWay { // 周围半径为3千米 def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) DBObject q = new BasicDBObject() - q.put('status', 1) + q.put('status', 0) q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); Map map = new ConcurrentHashMap<>(); - poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) if (distance <= 3000) { dealSpecial(obj, map, id, distance, result) @@ -84,8 +93,8 @@ class AroundExportByNewWay { }) }; latch.await() + def headArr = ['IDB', '名称','类型', '距离'] def ret = [] - ret << ['IDB', '名称','类型', '距离'] for (List district : data) { def district_code = district.get(2) def name = district.get(3) @@ -94,7 +103,9 @@ class AroundExportByNewWay { } } println(ret.size()) - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + FileUtils.batchCreateFile(ret, "F:\\数据\\重庆农商行\\周边配套.dat", headArr, "utf-8") +// EasyExcel.write("F:\\数据\\重庆农商行\\周边配套.xlsx").head(head).sheet("成交信息").doWrite(ret); executorService.shutdown() } @@ -143,6 +154,9 @@ class AroundExportByNewWay { return false } } + if (name.indexOf("(出入口)") != -1) { + return false + } } // 中梁山地铁站1A口 if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy index 76226f770..c9eb96812 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy @@ -4,7 +4,9 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import com.mongodb.QueryOperators +import sun.swing.BakedArrayList /** * 标记不需要导出的数据 @@ -23,8 +25,13 @@ class MarkUnNeedExport { if (district.size() > 1) { def type = district.get(0) def status = district.get(1) - if (status == 1) { - poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1))) + if (type == '交通设施服务;公交车站;公交车站相关') { + println(type) + } + if (status == 1.0) { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 0)), false, true) + } else { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index eef249210..721ab6958 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -4,6 +4,8 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.util.HttpRequestUtil import net.sf.json.JSONObject +import java.util.concurrent.ConcurrentHashMap + /** * 高德工具类 * @@ -14,6 +16,20 @@ class GaoDeUtils { def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988" + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + /** * * @param city From d0744e2df5258c380241f9a0a15797f74dd6fb1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 16 May 2021 19:29:35 +0800 Subject: [PATCH 039/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 9 +-- .../shenjian/cqrcb/MarkUnNeedExport.groovy | 3 - .../script/temp/shenjian/cqrcb/POIRule.groovy | 61 +++++++++++++++++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 5 +- .../temp/shenjian/util/GaoDeUtils.groovy | 14 ++++- 5 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 94e4b9342..564da168d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -30,12 +30,6 @@ class AroundExportByNewWay { static def resultMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) - static Map map = new HashedMap() - - static { - map.put("") - } - static void main(String[] args) { List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) AtomicInteger count = new AtomicInteger(0) @@ -119,6 +113,9 @@ class AroundExportByNewWay { return false } if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { + if (name.indexOf("栋") != -1) { + return false + } if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { if (name.indexOf("分院") == -1) { return false diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy index c9eb96812..9ce164947 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy @@ -25,9 +25,6 @@ class MarkUnNeedExport { if (district.size() > 1) { def type = district.get(0) def status = district.get(1) - if (type == '交通设施服务;公交车站;公交车站相关') { - println(type) - } if (status == 1.0) { poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 0)), false, true) } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy new file mode 100644 index 000000000..ca637b48e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.regex.Pattern + +/** + * POI清理规则 + * + * @author shenjian + * @since 2021/5/16 + */ +class POIRule { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + // 小学校转为学校 + poiConn.find(new BasicDBObject("typecode", "141203").append("status", 0)).batchSize(1000).each {obj -> + if (obj.name.indexOf('小学校') != -1 ) { + def id = obj._id + def region = obj.region + def name = obj.name.replace("小学校", '小学') + if (poiConn.count(new BasicDBObject("region": region, "name": name)) > 0) { + poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true) + } else { + poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("name", name)), false, true) + } + } + } + } + + static BasicDBObject getLikeStr(String findStr){ + Pattern pattern = Pattern.compile('^.*' + findStr + '.*$', Pattern.CASE_INSENSITIVE); + return new BasicDBObject('$regex', pattern); + } + +// 更新规则 +// db.getCollection('gaode_address_poi').update({'name': /栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /购物服务;商场;免税品店/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;口腔医院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;综合医院;卫生院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[学]+楼/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[教医光]+学中心/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;学校/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[1234]期/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /住院部/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /\//, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index feb17ea7f..69638ec6f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -85,6 +85,9 @@ class SeaweedExport { ret[0] = obj.developer ret[1] = obj.estate_company ret[2] = obj.cubage_percent + if (obj.developer == '重庆汇凯置业有限公司') { + println('重庆汇凯置业有限公司') + } ret[3] = virescencePercentValidator(obj.virescence_percent) if (obj.alias) { @@ -145,7 +148,7 @@ class SeaweedExport { if(value>=10 && value<=100){ return value; } else if (value > 0 && value < 1) { - return value * 100 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value) * 100) } else{ return ""; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 721ab6958..1ccbf1713 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -19,7 +19,19 @@ class GaoDeUtils { def static codeNameMap = new HashMap(){{ put("150500", "地铁"); put("1507", "公交"); - put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") put("141200", "学校") put("141201", "学校") put("141202", "学校") From 65b8edb112b14914dbfa8d78359f4cd94308be56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 10:17:12 +0800 Subject: [PATCH 040/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=BD=A6=E4=BD=8D?= =?UTF-8?q?=E6=AF=94=E5=8F=8A=E7=BB=BF=E5=8C=96=E7=8E=87=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/cqrcb/POIRule.groovy | 5 +++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index ca637b48e..e6ab78d3d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.cqrcb +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -18,6 +19,10 @@ class POIRule { static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static void main(String[] args) { + + } + + static void changeSchool() { // 小学校转为学校 poiConn.find(new BasicDBObject("typecode", "141203").append("status", 0)).batchSize(1000).each {obj -> if (obj.name.indexOf('小学校') != -1 ) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 69638ec6f..e5e264415 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.temp.shenjian.cqrcb import com.fangjia.core.util.DateUtil +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -85,9 +87,6 @@ class SeaweedExport { ret[0] = obj.developer ret[1] = obj.estate_company ret[2] = obj.cubage_percent - if (obj.developer == '重庆汇凯置业有限公司') { - println('重庆汇凯置业有限公司') - } ret[3] = virescencePercentValidator(obj.virescence_percent) if (obj.alias) { @@ -99,7 +98,17 @@ class SeaweedExport { ret[4] = temp.join(",") } ret[5] = obj.car_site_count - ret[6] = "" + if (com.fangjia.script.datastore.FieldValidatorUtils.householdCountValidator(obj.household_count) + && FieldValidatorUtils.carSiteValidator(obj.car_site_count)) { + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) + } + ret[6] = "1:" + val + } ret[7] = "" ret[8] = obj.all_area ret[9] = obj.loudong_count @@ -146,7 +155,7 @@ class SeaweedExport { def value = com.fangjia.script.datastore.DistrictFieldNumberic.checkValue(virescence_percent); if(value){ if(value>=10 && value<=100){ - return value; + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value)) } else if (value > 0 && value < 1) { return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value) * 100) } else{ From 84260e7d46687a69ffcbd3a29ecf548e21657941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 10:42:29 +0800 Subject: [PATCH 041/118] =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 155 +++++++----------- .../temp/shenjian/cqrcb/DealUnMatched.groovy | 106 ++++++++++++ .../temp/shenjian/cqrcb/ImportSchool.groovy | 53 ++++++ .../shenjian/cqrcb/MoveArroundTemp.groovy | 81 +++++++-- .../script/temp/shenjian/cqrcb/POIRule.groovy | 8 +- .../script/temp/shenjian/cqrcb/SavePoi.groovy | 55 +++++++ .../temp/shenjian/school/DistanceDeal.groovy | 13 +- .../temp/shenjian/util/GaoDeUtils.groovy | 19 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 32 ++++ 9 files changed, 398 insertions(+), 124 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 564da168d..5147a5b04 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -28,6 +28,7 @@ class AroundExportByNewWay { static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -42,9 +43,6 @@ class AroundExportByNewWay { try { def id = district.get(0) def district_code = district.get(2) - if (district_code == 'B100203') { - println(district_code) - } def name = district.get(3) List result = [] if (district.get(20) != null && district.get(21) != null) { @@ -57,24 +55,28 @@ class AroundExportByNewWay { q.put('status', 0) q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); - Map map = new ConcurrentHashMap<>(); poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) if (distance <= 3000) { - dealSpecial(obj, map, id, distance, result) + if (distance == 0) { + distance = 50 + } + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = formatType(obj.typecode) + ret[3] = distance.toInteger() + result << ret } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - def type = map.get(key).get("type") - def ret = [] - ret[0] = id - ret[1] = key - ret[2] = type - ret[3] = (distance / count1).toInteger() - result << ret - } + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) @@ -86,100 +88,63 @@ class AroundExportByNewWay { } }) }; - latch.await() + latch.await(); def headArr = ['IDB', '名称','类型', '距离'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + Map map = new HashedMap() + Integer key = 1; def ret = [] for (List district : data) { def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { ret.addAll(resultMap.get(district_code + name)) + if (ret.size() > 600000) { + map.put(key++, ret) + ret = [] + } } } - println(ret.size()) + map.put(key++, ret) //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); - FileUtils.batchCreateFile(ret, "F:\\数据\\重庆农商行\\周边配套.dat", headArr, "utf-8") -// EasyExcel.write("F:\\数据\\重庆农商行\\周边配套.xlsx").head(head).sheet("成交信息").doWrite(ret); + for (Integer k : map.keySet()) { + FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + //EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + //printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def id, double distance, def result) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("栋") != -1) { - return false - } - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false - } - } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("(出入口)") != -1) { - return false - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - distance + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - child.put("type", obj.type) - map.put(key, child) - } else { - child.put("distance", distance) - child.put("type", obj.type) - child.put("count", 1) - map.put(key, child) - } - } else { - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = obj.type - ret[3] = distance.toInteger() - result << ret + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String type) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy new file mode 100644 index 000000000..48be458a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +/** + * 处理未匹配的小区 + * + * @author shenjian + * @since 2021/5/17 + */ +class DealUnMatched { + + static void main(String[] args) { + // getLatLngByName() + // getLatLngByNumber() + //findExceptionDistance() + + // B114862 + def lng = 107.736591 + def lat = 29.872502 + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 500) + DBObject q = new BasicDBObject() + q.put('city', '重庆') + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + DBUtils.seaweed.find(q).batchSize(100).each {obj -> + println(obj) + } + } + + static void findExceptionDistance() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + for (List district : data) { + if (district.size() > 12) { + def lng = java.lang.Double.valueOf(district.get(8)) + def lat = java.lang.Double.valueOf(district.get(9)) + def lng1 = java.lang.Double.valueOf(district.get(11)) + def lat1 = java.lang.Double.valueOf(district.get(12)) + double distance = GaoDeUtils.getDistance(lat, lng, lat1, lng1) + if (distance > 500) { + println("距离异常" + district) + district.add(distance) + district.add("距离异常") + } + } + } + data.add(0, new ArrayList()) + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "周边配套") + } + + static void getLatLngByNumber() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",1) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + + //def region = SeaweedUtils.getRegionByCode(district_code) + if (district.size() <= 5) { + if (number != null && number.size() > 5) { + def map = GaoDeUtils.parserByGaode(city, number) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过楼栋得出") + } + } else { + println(district_code + " " + name + " " + number) + } + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配1") + } + + static void getLatLngByName() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", 0) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + //def region = SeaweedUtils.getRegionByCode(district_code) + + def map = GaoDeUtils.parserByGaode(city, name) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过小区得出") + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy new file mode 100644 index 000000000..b8f0452ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy @@ -0,0 +1,53 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 导入中学小学信息 + * + * @author shenjian + * @since 2021/5/17 + */ +class ImportSchool { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1) + List result = new ArrayList(); + for (List school : data) { + try { + def city = school.get(0) + def region = school.get(1) + def name = school.get(2) + def address = school.get(4) + def type = school.get(6) + def lng = school.get(9) + def lat = school.get(10) + + DBObject dbObject = new BasicDBObject() + dbObject.put("province", "重庆") + dbObject.put("city", city) + dbObject.put("region", region) + dbObject.put("name", name) + dbObject.put("address", address) + dbObject.put("type", type) + dbObject.put("typecode", "141200") + dbObject.put("lng", lng) + dbObject.put("lat", lat) + dbObject.put("status", 0) + dbObject.put("flag", "excel导入") + result.add(dbObject) + } catch(Exception ex) { + println(school) + ex.printStackTrace() + } + } + + poiConn.insert(result) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy index e17e12840..c641a4fc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy @@ -1,13 +1,17 @@ package com.fangjia.script.temp.shenjian.cqrcb +import cn.emay.mina.util.ConcurrentHashSet import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger /** * POI迁移 @@ -22,18 +26,20 @@ class MoveArroundTemp { static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - List list = Collections.synchronizedList(new ArrayList()) - List ids = conn.distinct("id") - CountDownLatch latch = new CountDownLatch(ids.size()) - for (String idT : ids) { - final String id = idT + Map map = new ConcurrentHashMap() + AtomicInteger count = new AtomicInteger() + int size = conn.count(new BasicDBObject("status", 1)) + println(size) + CountDownLatch latch = new CountDownLatch(size) + conn.find(new BasicDBObject("status", 1)).batchSize(10000).each { objT -> + final def obj = objT executorService.submit(new Runnable() { @Override void run() { try { - DBObject dbObject = new BasicDBObject() - conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> - dbObject.put("_id", id) + if (map.get(obj.id) == null && conn43.count(new BasicDBObject("_id", obj.id)) <= 0) { + DBObject dbObject = new BasicDBObject() + dbObject.put("_id", obj.id) dbObject.put("name", obj.name) dbObject.put("type", obj.type) dbObject.put("typecode", obj.typecode) @@ -44,21 +50,66 @@ class MoveArroundTemp { dbObject.put("province", obj.pname) dbObject.put("city", obj.cityname) dbObject.put("region", obj.adname) + dbObject.put("status", 2) + dbObject.put("flag", "待清洗") + map.put(obj.id, dbObject) } - list.add(dbObject) } catch(Exception ex) { ex.printStackTrace() } finally { latch.countDown() - println("当前记录数" + list.size()) + println(count.incrementAndGet()) } } }) - }; - latch.await(); - conn43.insert(list); - list.clear(); - println("同步清理完成"); + } + latch.await() + List ret = new ArrayList() + for (String key : map.keySet()) { + ret << map.get(key) + } + conn43.insert(ret) executorService.shutdown() } + + static origin() { +// List list = Collections.synchronizedList(new ArrayList()) +// List ids = conn.distinct("id") +// CountDownLatch latch = new CountDownLatch(ids.size()) +// for (String idT : ids) { +// final String id = idT +// executorService.submit(new Runnable() { +// @Override +// void run() { +// try { +// DBObject dbObject = new BasicDBObject() +// conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> +// dbObject.put("_id", id) +// dbObject.put("name", obj.name) +// dbObject.put("type", obj.type) +// dbObject.put("typecode", obj.typecode) +// dbObject.put("address", obj.address) +// dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) +// dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) +// dbObject.put("tel", obj.tel) +// dbObject.put("province", obj.pname) +// dbObject.put("city", obj.cityname) +// dbObject.put("region", obj.adname) +// } +// list.add(dbObject) +// } catch(Exception ex) { +// ex.printStackTrace() +// } finally { +// latch.countDown() +// println("当前记录数" + list.size()) +// } +// } +// }) +// }; +// latch.await(); +// conn43.insert(list); +// list.clear(); +// println("同步清理完成"); +// executorService.shutdown() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index e6ab78d3d..88033d0b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -44,10 +44,11 @@ class POIRule { } // 更新规则 -// db.getCollection('gaode_address_poi').update({'name': /栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); +// db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); @@ -63,4 +64,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +// +//db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false}); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy new file mode 100644 index 000000000..9da03d468 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author shenjian + * @since 2021/5/17 + */ +class SavePoi { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + AtomicInteger count = new AtomicInteger() + for (List listT : data) { + final list = listT + executorService.execute(new Runnable() { + @Override + void run() { + if (list.size() >= 17) { + def city = list.get(1) + def lng = Double.valueOf(list.get(11)) + def lat = Double.valueOf(list.get(12)) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("status", 1) + obj.put("flag", "距离异常入库") + } + conn.insert(arounds) + println(count.incrementAndGet()) + } + } + } + }) + }; + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy index 3037ee807..f5c6f3504 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -75,16 +75,5 @@ class DistanceDeal { // } } - static def parserByGaode(def city, def address) { - JSONObject json = GaoDeApiUtils.geo(city, address) - if (json && json.status == "1") { - def ret = json.geocodes - if (ret) { - def str = ret.get(0).location - def level = ret.get(0).level - def arrs = str.split(",") - return ["lng": arrs[0], "lat": arrs[1]] - } - } - } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 1ccbf1713..3e745dfd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import net.sf.json.JSONObject import java.util.concurrent.ConcurrentHashMap @@ -84,4 +86,21 @@ class GaoDeUtils { } return result } + + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..2f93a62e2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } +} From 964b25baa262e457816e1bc7338eea444a640f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 10:42:29 +0800 Subject: [PATCH 042/118] =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 155 +++++++----------- .../temp/shenjian/cqrcb/DealUnMatched.groovy | 106 ++++++++++++ .../temp/shenjian/cqrcb/ImportSchool.groovy | 53 ++++++ .../shenjian/cqrcb/MoveArroundTemp.groovy | 81 +++++++-- .../script/temp/shenjian/cqrcb/POIRule.groovy | 8 +- .../script/temp/shenjian/cqrcb/SavePoi.groovy | 55 +++++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 3 +- .../temp/shenjian/school/DistanceDeal.groovy | 13 +- .../temp/shenjian/util/GaoDeUtils.groovy | 19 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 32 ++++ 10 files changed, 400 insertions(+), 125 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 564da168d..5147a5b04 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -28,6 +28,7 @@ class AroundExportByNewWay { static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -42,9 +43,6 @@ class AroundExportByNewWay { try { def id = district.get(0) def district_code = district.get(2) - if (district_code == 'B100203') { - println(district_code) - } def name = district.get(3) List result = [] if (district.get(20) != null && district.get(21) != null) { @@ -57,24 +55,28 @@ class AroundExportByNewWay { q.put('status', 0) q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); - Map map = new ConcurrentHashMap<>(); poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) if (distance <= 3000) { - dealSpecial(obj, map, id, distance, result) + if (distance == 0) { + distance = 50 + } + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = formatType(obj.typecode) + ret[3] = distance.toInteger() + result << ret } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - def type = map.get(key).get("type") - def ret = [] - ret[0] = id - ret[1] = key - ret[2] = type - ret[3] = (distance / count1).toInteger() - result << ret - } + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) @@ -86,100 +88,63 @@ class AroundExportByNewWay { } }) }; - latch.await() + latch.await(); def headArr = ['IDB', '名称','类型', '距离'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + Map map = new HashedMap() + Integer key = 1; def ret = [] for (List district : data) { def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { ret.addAll(resultMap.get(district_code + name)) + if (ret.size() > 600000) { + map.put(key++, ret) + ret = [] + } } } - println(ret.size()) + map.put(key++, ret) //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); - FileUtils.batchCreateFile(ret, "F:\\数据\\重庆农商行\\周边配套.dat", headArr, "utf-8") -// EasyExcel.write("F:\\数据\\重庆农商行\\周边配套.xlsx").head(head).sheet("成交信息").doWrite(ret); + for (Integer k : map.keySet()) { + FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + //EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + //printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def id, double distance, def result) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("栋") != -1) { - return false - } - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false - } - } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("(出入口)") != -1) { - return false - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - distance + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - child.put("type", obj.type) - map.put(key, child) - } else { - child.put("distance", distance) - child.put("type", obj.type) - child.put("count", 1) - map.put(key, child) - } - } else { - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = obj.type - ret[3] = distance.toInteger() - result << ret + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String type) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy new file mode 100644 index 000000000..48be458a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +/** + * 处理未匹配的小区 + * + * @author shenjian + * @since 2021/5/17 + */ +class DealUnMatched { + + static void main(String[] args) { + // getLatLngByName() + // getLatLngByNumber() + //findExceptionDistance() + + // B114862 + def lng = 107.736591 + def lat = 29.872502 + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 500) + DBObject q = new BasicDBObject() + q.put('city', '重庆') + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + DBUtils.seaweed.find(q).batchSize(100).each {obj -> + println(obj) + } + } + + static void findExceptionDistance() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + for (List district : data) { + if (district.size() > 12) { + def lng = java.lang.Double.valueOf(district.get(8)) + def lat = java.lang.Double.valueOf(district.get(9)) + def lng1 = java.lang.Double.valueOf(district.get(11)) + def lat1 = java.lang.Double.valueOf(district.get(12)) + double distance = GaoDeUtils.getDistance(lat, lng, lat1, lng1) + if (distance > 500) { + println("距离异常" + district) + district.add(distance) + district.add("距离异常") + } + } + } + data.add(0, new ArrayList()) + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "周边配套") + } + + static void getLatLngByNumber() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",1) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + + //def region = SeaweedUtils.getRegionByCode(district_code) + if (district.size() <= 5) { + if (number != null && number.size() > 5) { + def map = GaoDeUtils.parserByGaode(city, number) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过楼栋得出") + } + } else { + println(district_code + " " + name + " " + number) + } + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配1") + } + + static void getLatLngByName() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", 0) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + //def region = SeaweedUtils.getRegionByCode(district_code) + + def map = GaoDeUtils.parserByGaode(city, name) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过小区得出") + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy new file mode 100644 index 000000000..b8f0452ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy @@ -0,0 +1,53 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 导入中学小学信息 + * + * @author shenjian + * @since 2021/5/17 + */ +class ImportSchool { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1) + List result = new ArrayList(); + for (List school : data) { + try { + def city = school.get(0) + def region = school.get(1) + def name = school.get(2) + def address = school.get(4) + def type = school.get(6) + def lng = school.get(9) + def lat = school.get(10) + + DBObject dbObject = new BasicDBObject() + dbObject.put("province", "重庆") + dbObject.put("city", city) + dbObject.put("region", region) + dbObject.put("name", name) + dbObject.put("address", address) + dbObject.put("type", type) + dbObject.put("typecode", "141200") + dbObject.put("lng", lng) + dbObject.put("lat", lat) + dbObject.put("status", 0) + dbObject.put("flag", "excel导入") + result.add(dbObject) + } catch(Exception ex) { + println(school) + ex.printStackTrace() + } + } + + poiConn.insert(result) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy index e17e12840..c641a4fc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy @@ -1,13 +1,17 @@ package com.fangjia.script.temp.shenjian.cqrcb +import cn.emay.mina.util.ConcurrentHashSet import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger /** * POI迁移 @@ -22,18 +26,20 @@ class MoveArroundTemp { static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - List list = Collections.synchronizedList(new ArrayList()) - List ids = conn.distinct("id") - CountDownLatch latch = new CountDownLatch(ids.size()) - for (String idT : ids) { - final String id = idT + Map map = new ConcurrentHashMap() + AtomicInteger count = new AtomicInteger() + int size = conn.count(new BasicDBObject("status", 1)) + println(size) + CountDownLatch latch = new CountDownLatch(size) + conn.find(new BasicDBObject("status", 1)).batchSize(10000).each { objT -> + final def obj = objT executorService.submit(new Runnable() { @Override void run() { try { - DBObject dbObject = new BasicDBObject() - conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> - dbObject.put("_id", id) + if (map.get(obj.id) == null && conn43.count(new BasicDBObject("_id", obj.id)) <= 0) { + DBObject dbObject = new BasicDBObject() + dbObject.put("_id", obj.id) dbObject.put("name", obj.name) dbObject.put("type", obj.type) dbObject.put("typecode", obj.typecode) @@ -44,21 +50,66 @@ class MoveArroundTemp { dbObject.put("province", obj.pname) dbObject.put("city", obj.cityname) dbObject.put("region", obj.adname) + dbObject.put("status", 2) + dbObject.put("flag", "待清洗") + map.put(obj.id, dbObject) } - list.add(dbObject) } catch(Exception ex) { ex.printStackTrace() } finally { latch.countDown() - println("当前记录数" + list.size()) + println(count.incrementAndGet()) } } }) - }; - latch.await(); - conn43.insert(list); - list.clear(); - println("同步清理完成"); + } + latch.await() + List ret = new ArrayList() + for (String key : map.keySet()) { + ret << map.get(key) + } + conn43.insert(ret) executorService.shutdown() } + + static origin() { +// List list = Collections.synchronizedList(new ArrayList()) +// List ids = conn.distinct("id") +// CountDownLatch latch = new CountDownLatch(ids.size()) +// for (String idT : ids) { +// final String id = idT +// executorService.submit(new Runnable() { +// @Override +// void run() { +// try { +// DBObject dbObject = new BasicDBObject() +// conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> +// dbObject.put("_id", id) +// dbObject.put("name", obj.name) +// dbObject.put("type", obj.type) +// dbObject.put("typecode", obj.typecode) +// dbObject.put("address", obj.address) +// dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) +// dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) +// dbObject.put("tel", obj.tel) +// dbObject.put("province", obj.pname) +// dbObject.put("city", obj.cityname) +// dbObject.put("region", obj.adname) +// } +// list.add(dbObject) +// } catch(Exception ex) { +// ex.printStackTrace() +// } finally { +// latch.countDown() +// println("当前记录数" + list.size()) +// } +// } +// }) +// }; +// latch.await(); +// conn43.insert(list); +// list.clear(); +// println("同步清理完成"); +// executorService.shutdown() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index e6ab78d3d..88033d0b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -44,10 +44,11 @@ class POIRule { } // 更新规则 -// db.getCollection('gaode_address_poi').update({'name': /栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); +// db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); @@ -63,4 +64,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +// +//db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false}); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy new file mode 100644 index 000000000..9da03d468 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author shenjian + * @since 2021/5/17 + */ +class SavePoi { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + AtomicInteger count = new AtomicInteger() + for (List listT : data) { + final list = listT + executorService.execute(new Runnable() { + @Override + void run() { + if (list.size() >= 17) { + def city = list.get(1) + def lng = Double.valueOf(list.get(11)) + def lat = Double.valueOf(list.get(12)) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("status", 1) + obj.put("flag", "距离异常入库") + } + conn.insert(arounds) + println(count.incrementAndGet()) + } + } + } + }) + }; + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index e5e264415..66703a1f0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -95,7 +95,8 @@ class SeaweedExport { for (def a : obj.alias) { temp.add(a) } - ret[4] = temp.join(",") + String als = temp.join(",") + ret[4] = als.length() > 20 ? "" : als } ret[5] = obj.car_site_count if (com.fangjia.script.datastore.FieldValidatorUtils.householdCountValidator(obj.household_count) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy index 3037ee807..f5c6f3504 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -75,16 +75,5 @@ class DistanceDeal { // } } - static def parserByGaode(def city, def address) { - JSONObject json = GaoDeApiUtils.geo(city, address) - if (json && json.status == "1") { - def ret = json.geocodes - if (ret) { - def str = ret.get(0).location - def level = ret.get(0).level - def arrs = str.split(",") - return ["lng": arrs[0], "lat": arrs[1]] - } - } - } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 1ccbf1713..3e745dfd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import net.sf.json.JSONObject import java.util.concurrent.ConcurrentHashMap @@ -84,4 +86,21 @@ class GaoDeUtils { } return result } + + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..2f93a62e2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } +} From 034f924f509720dd9c16ff8122a0619e3ecb5b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 18:39:41 +0800 Subject: [PATCH 043/118] =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 65 ++++++++++++------- .../script/temp/shenjian/cqrcb/POIRule.groovy | 2 +- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 5147a5b04..4ec4dac97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -4,6 +4,7 @@ import com.alibaba.excel.EasyExcel import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.district.Distance import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -45,28 +46,23 @@ class AroundExportByNewWay { def district_code = district.get(2) def name = district.get(3) List result = [] - if (district.get(20) != null && district.get(21) != null) { + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) - // 周围半径为3千米 - def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) - DBObject q = new BasicDBObject() - q.put('status', 0) - q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); - q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); - poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> - double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) - if (distance <= 3000) { - if (distance == 0) { - distance = 50 - } - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = formatType(obj.typecode) - ret[3] = distance.toInteger() - result << ret + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id) } } } else { @@ -81,7 +77,7 @@ class AroundExportByNewWay { resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) } catch (Exception ex) { - + ex.printStackTrace() } finally { latch.countDown() } @@ -113,14 +109,37 @@ class AroundExportByNewWay { map.put(key++, ret) //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); for (Integer k : map.keySet()) { - FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") - //EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); } - //printUnMatched(unMatchedMap) + printUnMatched(unMatchedMap) executorService.shutdown() } + static calcAroundInfo(def lat, def lng, def result, def id) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 3000) { + if (distance == 0) { + distance = 50 + } + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = formatType(obj.typecode) + ret[3] = distance.toInteger() + result << ret + } + } + } + static printUnMatched(def unMatchedMap) { def ret = [] ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index 88033d0b8..5deadb989 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -48,7 +48,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[ABCD]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCDEFHG]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); From 113f2e8c506154773725fc21e51f431b0d19ad5e Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Mon, 17 May 2021 20:58:57 +0800 Subject: [PATCH 044/118] =?UTF-8?q?=E5=B0=8F=E5=8C=BAmaster=E5=88=86?= =?UTF-8?q?=E6=94=AF=E5=BC=80=E5=8F=91=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/company/AddCompany.groovy | 71 +++++ .../shenjian/company/DevelopCompany.groovy | 33 ++ .../shenjian/company/PropertyCompany.groovy | 34 +++ .../temp/shenjian/dto/SearchAddressDTO.groovy | 16 + .../fill/CalcCubageAndVirescence.groovy | 116 +++++++ .../shenjian/seaweed/BuildStructure.groovy | 112 +++++++ .../temp/shenjian/seaweed/BuildType.groovy | 129 ++++++++ .../temp/shenjian/seaweed/BuildYear.groovy | 91 ++++++ .../temp/shenjian/seaweed/CollectTest.groovy | 87 ++++++ .../temp/shenjian/seaweed/Company.groovy | 155 ++++++++++ .../script/temp/shenjian/seaweed/Fee.groovy | 112 +++++++ .../temp/shenjian/seaweed/HousePurpose.groovy | 109 +++++++ .../temp/shenjian/seaweed/HouseType.groovy | 99 ++++++ .../temp/shenjian/seaweed/Position.groovy | 78 +++++ .../temp/shenjian/seaweed/Seaweed.groovy | 288 ++++++++++++++++++ .../script/temp/shenjian/seaweed/Temp.groovy | 28 ++ .../temp/shenjian/util/DistrictUtils.groovy | 55 ++++ 17 files changed, 1613 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy new file mode 100644 index 000000000..1c8d2f70f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils; + +/** + * + * 新增公司 + * + * @author shenjian + * @since 2021-03-16 + */ +class AddCompany { + + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static final Set developCompanySet = new HashSet() + static final Set propertyCompanySet = new HashSet() + + static final Set actualDevelopCompanySet = new HashSet() + static final Set actualPropertyCompanySet = new HashSet() + + static { + develop_company.find().batchSize(10000).each { obj -> + developCompanySet.add(obj.company) + } + property_company.find().batchSize(10000).each { obj -> + propertyCompanySet.add(obj.company) + } + } + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "哈尔滨") + DbUtils.seaweed.find(query, + new BasicDBObject("developer", 1).append("estate_company", 1)) + .batchSize(10000).each {obj -> + actualDevelopCompanySet.add(obj.developer) + actualPropertyCompanySet.add(obj.estate_company) + } + + def developCompanySaves = [] + def propertyCompanySaves = [] + for (String name : actualDevelopCompanySet) { + if (StringUtils.isBlank(name) || developCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + developCompanySaves << item + } + for (String name : actualPropertyCompanySet) { + if (StringUtils.isBlank(name) || propertyCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + propertyCompanySaves << item + } + + + develop_company.insert(developCompanySaves) + property_company.insert(propertyCompanySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy new file mode 100644 index 000000000..fb1e94c2b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils; + +/** + * + * 开发商公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class DevelopCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static void main(String[] args) { + def companySaves = [] + List result = seaweed.distinct("developer") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + develop_company.insert(companySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy new file mode 100644 index 000000000..dca212b14 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy @@ -0,0 +1,34 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils + +/** + * 物业公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class PropertyCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static void main(String[] args) { + // 物业公司 + def companySaves = [] + List result = seaweed.distinct("estate_company") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + property_company.insert(companySaves) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy new file mode 100644 index 000000000..01b1c1f27 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy @@ -0,0 +1,16 @@ +package com.fangjia.script.temp.shenjian.dto + +/** + * 匹配返回结构体 + */ +class SearchAddressDTO { + + String code; + String msg; + Result result; + + class Result { + String street + String streetNumber + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy new file mode 100644 index 000000000..ada6e667b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.fill + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.GeoHelper +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 根据周围500米容积率与绿化率平均值推算小区 + * + * @author shenjian* + * @since 2021-03-13 + */ +@Slf4j +class CalcCubageAndVirescence { + + static main(args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("CalcCubageAndVirescence") + log.info("开始根据周围500米容积率与绿化率平均值推算小区") + def cities = ['北京', '重庆', '哈尔滨'] + for (def city : cities) { + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts); + def result = [] + districts.each { obj -> + if (obj.finish_building_date) { + int year = CommonUtils.getBuildYear(obj.finish_building_date) + if (year > 2000) { + if (!(obj.cubage_percent && obj.virescence_percent)) { + def temp = this.findResult(obj.lat2 / 10000000000.0d, obj.lng2 / 10000000000.0d, 500, sameRegionDistricts.get(obj.region), obj.name, obj._id) + result << temp + } + + } + } + } + def headers = ["小区ID", "容积率", "绿化率"] + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\补充容积绿化" + city + ".csv", headers, "UTF-8") + } + stopWatch.stop() + log.info("完成根据周围500米容积率与绿化率平均值推算, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + } + + static def findResult(lat, lng, radius, districts, region, id) { + def avgMap = [:] + def ret = [id] + if (lat && lng) { + def cubagePercentArr = [] + def virescencePercentArr = [] + def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, radius) + for (def row : districts) { + if (row.lat2 && row.lng2) { + def lat2 = row.lat2 / 10000000000.0; + def lng2 = row.lng2 / 10000000000.0; + if (lat2 >= map.minLat && lat2 <= map.maxLat && lng2 >= map.minLng && lng2 <= map.maxLng) { + def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2) + if (distance <= radius) { + if (row.cubage_percent) { + cubagePercentArr << row.cubage_percent; + } + if (row.virescence_percent) { + virescencePercentArr << row.virescence_percent + } + } + } + } + } + try { + // 计算平均容积率 + if (cubagePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : cubagePercentArr) { + sum = sum + Double.valueOf(m) + count++ + } + if (sum) { + double avgCubagePercent = sum / count + ret << DataHandleUtils.getDoubleRound(2, avgCubagePercent) + } + } else { + ret << "" + } + + // 计算平均绿化率 + if (virescencePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : virescencePercentArr) { + sum += Double.valueOf(m) + count++ + } + if (sum) { + def avgVirescencePercent = sum / (count) + ret << DataHandleUtils.getDoubleRound(2, avgVirescencePercent) + } + } else { + ret << "" + } + log.info("找到可以优化的小区" + region + "周围平均值为:容积率" + ret[1] + ", 绿化率" + ret[2]) + } catch(Exception ex) { + log.error("容积率或绿化率不标准,无法解析") + } + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy new file mode 100644 index 000000000..278446f50 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑结构 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildStructure { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildStructureMap = new HashMap() + static { + buildStructureMap.put("钢结构", "3") + buildStructureMap.put("钢、钢筋混凝土结构", "1") + buildStructureMap.put("钢筋混凝土结构", "1") + buildStructureMap.put("钢混", "1") + buildStructureMap.put("钢混结构", "1") + buildStructureMap.put("混合结构", "11") + buildStructureMap.put("钢筋混凝土", "11") + buildStructureMap.put("砖木结构", "10") + buildStructureMap.put("其他结构", "11") + buildStructureMap.put("砖混结构", "2") + buildStructureMap.put("框架结构", "4") + buildStructureMap.put("框架剪力墙结构", "5") + buildStructureMap.put("剪力墙结构", "6") + buildStructureMap.put("筒体结构", "7") + buildStructureMap.put("框架筒体结构", "8") + buildStructureMap.put("筒中筒", "9") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_structure", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑结构","备注", "创建时间", "更新时间", "来源公司"] + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑结构\\bigdata_commbuildstru_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑结构表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getBuildStructure(dbo.get("build_structure")) // 建筑结构 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("build_structure"))) { + rows << row + count.incrementAndGet() + } + }) + return rows + }; + + static def getBuildStructure(def houseStructure) { + def ret + if (StringUtils.isNotBlank(houseStructure)) { + ret = buildStructureMap.get(houseStructure) + if (StringUtils.isBlank(ret)) { + ret = "11" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy new file mode 100644 index 000000000..408b87f48 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy @@ -0,0 +1,129 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildTypeMap = new HashMap() + static { + buildTypeMap.put("多层", "0104") + buildTypeMap.put("小高层", "0103") + buildTypeMap.put("高层", "0199") + buildTypeMap.put("超高层", "0199") + buildTypeMap.put("花园洋房", "0199") + buildTypeMap.put("老洋房", "0199") + buildTypeMap.put("四合院", "0199") + buildTypeMap.put("独栋", "0105") + buildTypeMap.put("双拼", "0199") + buildTypeMap.put("联排", "0106") + buildTypeMap.put("叠加", "0107") + buildTypeMap.put("老式别墅洋房", "0199") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑类型","备注", "创建时间", "更新时间", "来源公司"]; + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑类型\\bigdata_commbuildtype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + if (StringUtils.isNotBlank(dbo.get("build_type"))) { + Set set = this.getBuildTypeCode(dbo.get("build_type")) + for (String type : set) { + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << type // 建筑类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + } + if (set.size() > 0) { + count.incrementAndGet() + } + + } + }) + return rows + } + + + static Set getBuildTypeCode(def buildType) { + Set set = new HashSet<>() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + set.add(buildTypeMap.get(type)) + } + if (set.isEmpty()) { + set.add("0199") // 其他 + } + } + return set + } + + static List getBuildTypeCodeSeq(def buildType) { + List list = new ArrayList() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + list.add(buildTypeMap.get(type)) + } + if (list.isEmpty()) { + list.add("0199") // 其他 + } + } + return list + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy new file mode 100644 index 000000000..cee15c27c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.text.SimpleDateFormat +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑年代表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildYear { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildYear") + log.info("开始导出小区建筑年代表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("finish_building_date", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑年代","备注", "创建时间", "更新时间", "来源公司"] + + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑年代\\bigdata_commbuildyear_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑年代表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + Integer year = CommonUtils.getBuildYear(dbo.get("finish_building_date")) // 建筑年代 + if (year > 1900) { + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << year + row << "" // 备注 + + Integer createYear = CommonUtils.getBuildYear(DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy")) + if (year > createYear) { + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()); + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()) + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } + row << "fjw" // 来源公司 + + rows << row + count.incrementAndGet() + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy new file mode 100644 index 000000000..e5cbce7af --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy @@ -0,0 +1,87 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.ExcelUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * @author shenjian + * @since 2021-03-13 + */ +class CollectTest { + + static void main(String[] args) { + def cites = ["北京", "重庆", "哈尔滨"]; + for (def city : cites) { + /** + * _id 小区ID + * name 小区名 + * alias 小区曾用名 + */ + def districts = this.findAllCityDistricts(city, "district", + ["city", "_id", "region", "name", "alias"]) + print(districts) + } + } + + static def findAllCityDistricts(def city, def cat, def fields) { + DBObject q = new BasicDBObject(); + q.put("visible", 0) + q.put("cat", cat) + q.put("region", "密云") + if (city) { + q.put("city", city) + } + + DBObject keys = new BasicDBObject(); + if (fields) { + for (def field : fields) { + keys.put(field, 1) + } + } + + DSCursor cursor = CommonDatautils.getDataStoreConnection().find(q, keys).batchSize(4000) + def districtKeys = [] + def rows = []; + rows << ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + + for (def dbo : cursor) { + def row = []; + row << null // TODO + row << dbo.get("_id") + row << dbo.get("name") + row << dbo.get("alias") + row << null // TODO + row << dbo.get("street_community") // 街道 TODO 拆分为街道门牌号 + row << dbo.get("street_community") // 门牌号 + rows << dbo.get("address") + rows << dbo.get("all_area") + rows << dbo.get("build_area") + + rows << dbo.get("land_property") + ExcelUtils.write("d:\\数据\\小区表.xlsx", rows, "小区") + + StringBuilder data = new StringBuilder() + Set keySet = dbo.keySet() + keySet.forEach({key -> + if ("alias" == key) { + def value = dbo.get(key) + if (value != null) { + value = value[0] + } + data.append(value) + } else { + data.append(dbo.get(key)) + } + data.append("|@|") + }) + districtKeys << StringUtils.substringBeforeLast(data.toString(), "|@|") + }; + return districtKeys; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy new file mode 100644 index 000000000..87b609daa --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang3.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区公司表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Company { + + private static ExecutorService executorService = Executors.newFixedThreadPool(2); + + // 物业公司 + static final DBCollection PROPERTY_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + // 开发商 + static final DBCollection DEVELOP_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static final propertyCompany = new HashMap() + static final developCompany = new HashMap() + + static { + PROPERTY_COMPANY_COLLECTION.find().forEach({company -> + propertyCompany.put(company.get("company"), company.get("_id").toString()) + }) + + DEVELOP_COMPANY_COLLECTION.find().forEach({company -> + developCompany.put(company.get("company"), company.get("_id").toString()) + }) + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Company") + log.info("开始导出小区公司表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("developer", 1) + .append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","公司ID","公司类型", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger developCount = new AtomicInteger(0) + AtomicInteger estateCount = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, developCount, estateCount) + println("城市" + city + " " + cityId + "物业数量为" + estateCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((estateCount.get() * 100) / seaweedCount[i])) ) + println("城市" + city + " " + cityId + "开发商数量为" + developCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((developCount.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\公司\\bigdata_commcomp_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区公司表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger developCount, AtomicInteger estateCount) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + def rowCanSave = true + def row1CanSave = true + row << cityId // 城市ID + row << dbo.get("_id").toString() + def propertyCompany = dbo.get("estate_company") + if (StringUtils.isNotBlank(propertyCompany)) { + row << this.getPropertyCompanyId(propertyCompany) // 公司ID + row << "SV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + rowCanSave = false + } + + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + def row1 = [] + row1 << cityId + row1 << dbo.get("_id").toString() + def developerCompany = dbo.get("developer") + if (StringUtils.isNotBlank(developerCompany)) { + row1 << this.getDevelopCompanyId(developerCompany) // 公司ID + row1 << "DV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + row1CanSave = false + } + + row1 << "" // 备注 + row1 << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row1 << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row1 << "fjw" // 来源公司 + if (rowCanSave) { + rows << row + estateCount.incrementAndGet() + } + if (row1CanSave) { + rows << row1 + developCount.incrementAndGet() + } + }) + return rows + } + + /** + * 获取开发商公司ID + * @param company 公司名称 + * @return + */ + static def getDevelopCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return developCompany.get(company) + } + + /** + * 获取物业公司ID + * @param company 公司名称 + * @return + */ + static def getPropertyCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return propertyCompany.get(company) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy new file mode 100644 index 000000000..614da8edb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区费用表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Fee { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Fee") + log.info("开始导出小区费用表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("estate_charge", 1) + .append("estate_type2", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID", "公司ID", "费用", "房屋用途","建筑类型", "小区费用类型", "费用说明", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\费用\\bigdata_commfee_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区费用表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def estate_charge = dbo.get("estate_charge") + if (estate_charge && estate_charge.toString().indexOf("~") != -1) { + String[] arr = estate_charge.toString().split("~") + estate_charge = ((Double.valueOf(arr[0]) + Double.valueOf(arr[1])) / 2.0D).toString() + } + if (estate_charge == '-' || estate_charge == '无') { + estate_charge = "" + } + if (StringUtils.isNotBlank(dbo.get("estate_company")) && StringUtils.isNotBlank(estate_charge) + && StringUtils.isNotBlank(dbo.get("estate_type2")) && StringUtils.isNotBlank(dbo.get("build_type"))) { + List buildTypes = BuildType.getBuildTypeCodeSeq(dbo.get("build_type")) + Set savedBuildTypeSet = new HashSet<>() + boolean isFirst = true + for (String buildType : buildTypes) { + if (savedBuildTypeSet.contains(buildType)) { + continue + } + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() // 小区ID + row << Company.getPropertyCompanyId(dbo.get("estate_company")) // 公司ID + + // 处理 db.getCollection('seaweed').find({'estate_charge': '3.3~4.3'}, {'estate_charge': 1}) + // 特殊费用 3.3~4.3 + if (isFirst) { + row << estate_charge // 费用 + isFirst = false + count.incrementAndGet() + } else { + row << "" + break + } + row << HousePurpose.getHousePurposeCode(dbo.get("estate_type2")) // 房屋用途 + row << buildType // 建筑类型 + row << "1" // 小区费用类型 物业费1 电梯费2 垃圾费3 + row << dbo.get("") // 费用说明 + row << dbo.get("") // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + + savedBuildTypeSet.add(buildType) + } + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy new file mode 100644 index 000000000..ba8dc7c2e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy @@ -0,0 +1,109 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.StringUtil +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋用途表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HousePurpose { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map housePurposeMap = new HashMap() + static { + housePurposeMap.put("普通住宅", "11") + housePurposeMap.put("别墅", "11") + housePurposeMap.put("商住", "13") + housePurposeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区房屋用途表表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("household_count", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","房屋用途","户数", "备注", "创建时间", "更新时间", "来源公司"]; + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋用途\\bigdata_commhouseuse_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋用途表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << getHousePurposeCode(dbo.get("estate_type2")) + + if (dbo.get("household_count") != null) { + if (Long.valueOf(dbo.get("household_count")) < 50) { + row << "" // 户数 + } else { + row << dbo.get("household_count") + } + } else { + row << "" // 户数 + } + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHousePurposeCode(def housePurpose) { + def ret + if (StringUtils.isNotBlank(housePurpose)) { + ret = housePurposeMap.get(housePurpose) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy new file mode 100644 index 000000000..8e77760a9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HouseType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map houseTypeMap = new HashMap() + static { + houseTypeMap.put("普通住宅", "01") + houseTypeMap.put("商住", "07") + houseTypeMap.put("别墅", "08") + houseTypeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("HouseType") + log.info("开始导出小区房屋类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","小区房屋类型","备注", "创建时间", "更新时间", "来源公司"] + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋类型\\bigdata_commhousetype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getHouseTypeCode(dbo.get("estate_type2")) // 小区房屋类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHouseTypeCode(def houseType) { + def ret + if (StringUtils.isNotBlank(houseType)) { + ret = houseTypeMap.get(houseType) // 房屋类型 + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy new file mode 100644 index 000000000..269f01ab3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区地理位置表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Position { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Location") + log.info("开始导出小区地理位置表结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("lng", 1).append("lat", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","物体类型","物体ID","位置类型", "序号", "经度", "纬度", "海拔", "备注", "创建时间", "更新时间", "来源公司"] + + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\地理位置\\bigdata_position_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区地理位置表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << "5" // 物体类型 小区5 + row << dbo.get("_id").toString() // 物体ID + row << "1" // 位置类型 中心点1 + row << "1" // 序号 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lng") / 10000000000d) // 经度 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lat") / 10000000000d) // 纬度 + row << "" // 海拔 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + rows << row + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy new file mode 100644 index 000000000..0c2f16c13 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy @@ -0,0 +1,288 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.dto.SearchAddressDTO +import com.fangjia.script.temp.shenjian.fill.CalcCubageAndVirescence +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.regex.Pattern + +/** + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Seaweed { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + private static HttpRequest http = new HttpRequest() + private static final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + private static Map seaweedTypeMap = new HashMap(); + private static Pattern pattern = Pattern.compile("\\s*|\t|\r|\n"); // 取消空格换行等 + private static Map cityDistrictMap = new HashMap(); + + static { + seaweedTypeMap.put("普通住宅", "pa101") + seaweedTypeMap.put("别墅", "pa101") + seaweedTypeMap.put("商住", "pa101") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Seaweed") + log.info("开始导出小区公司表") + //def cites = ["北京", "重庆", "哈尔滨"] + //def seaweedCount = [21596, 17053, 5410]; + def cites = ["哈尔滨"] + def seaweedCount = [5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("name", 1).append("alias", 1).append("street_community", 1) + .append("address", 1).append("all_area", 1).append("build_area", 1).append("region", 1) + .append("estate_type2", 1).append("virescence_percent", 1).append("cubage_percent", 1) + .append("car_site_count", 1).append("c_date", 1).append("m_date", 1) + .append("lng", 1).append("lat", 1) + def headers = ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + final t = i; + def city = cites[t] + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts) + BasicDBObject cityDistrictInfo = FangJiaAreaMapUtils.convertToOfficial(city) + def cityId = cityDistrictInfo.get("city_code") // 城市ID + final def map = this.findAllCityDistricts(city, "district", cityId, filter, sameRegionDistricts) + executorService.execute(new Runnable() { + @Override + void run() { + def result = [] + AtomicInteger virescenceCount = new AtomicInteger(0) // 绿化率 + AtomicInteger cubageCount = new AtomicInteger(0) // 容积率 + AtomicInteger allAreaCount = new AtomicInteger(0) + AtomicInteger buildAreaCount = new AtomicInteger(0) + for (def key : map.keys()) { + result << map.get(key) + if (map.get(key) && map.get(key)[8] != "") { + allAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[9] != "") { + buildAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[12] != "") { + virescenceCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[13] != "") { + cubageCount.incrementAndGet() + } + } + println("城市" + city + " " + cityId + "绿化率数量为" + virescenceCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((virescenceCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "容积率数量为" + cubageCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((cubageCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "占地面积数量为" + allAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((allAreaCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "建筑面积数量为" + buildAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((buildAreaCount.get() * 100) / seaweedCount[t])) ) + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\bigdata_community_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + fixedThreadPool.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, def sameRegionDistricts) { + def map = new ConcurrentHashMap(); + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + def total = DbUtils.seaweed.find(query, filter).count() + final CountDownLatch latch = new CountDownLatch(total) + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + fixedThreadPool.execute(new Runnable() { + @Override + void run() { + try { + dealSpecialChar(dbo) + def row = [] + + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << dbo.get("name") + if (dbo.get("alias") != null && dbo.get("alias")[0] != null) { + row << pattern.matcher(dbo.get("alias")[0]).replaceAll("") + } else { + row << "" + } + def districtId + if (StringUtils.isBlank(cityDistrictMap.get(dbo.get("region")))) { + districtId = FangJiaAreaMapUtils.convertToOfficial(city, dbo.get("region")).get("district_code") + // 行政区ID + cityDistrictMap.put(dbo.get("region"), districtId) + } else { + districtId = cityDistrictMap.get(dbo.get("region")) + } + row << districtId + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city); + params.put("address", dbo.get("address")); + // 暂时不用解析 +// String s = http.get("http://open.fangjia.com/address/standardize", params, "utf-8"); +// SearchAddressDTO searchAddress = com.alibaba.fastjson.JSONObject.parseObject(s, SearchAddressDTO.class) +// if (searchAddress.code == '200') { +// //row << searchAddress.result.street // 街道 +// row << "" +// row << searchAddress.result.streetNumber // 门牌号 +// } else { +// row << "" +// row << "" +// } + row << "" // 街道 + row << "" // 门牌号 + + row << dbo.get("address") + + if (dbo.get("all_area") && dbo.get("all_area").toString().indexOf("公顷") != -1) { + def val = dbo.get("all_area").toString() + def temp = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000).longValue() + if (temp > 100000) { + row << "" + } else { + row << temp + } + + } else { + if (dbo.get("all_area")) { + if (Double.valueOf(dbo.get("all_area")) > 100000) { + row << "" + } else { + row << dbo.get("all_area") + } + } else { + row << "" + } + } + if (dbo.get("build_area") && Double.valueOf(dbo.get("build_area")) > 1000) { + row << dbo.get("build_area") + } else { + row << "" + } + + row << "pa101" // 小区类型 + row << "" // 是否新盘 + + if (dbo.get("virescence_percent") != null && dbo.get("cubage_percent") != null) { + // 绿化率 + if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) < 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 100 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) == 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 10 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) >= 70) { + row << 65 + } else { + row << Double.valueOf(dbo.get("virescence_percent")) + } + + if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 7 && Double.valueOf(dbo.get("cubage_percent")) < 8)) { + row << 6.2 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 8 && Double.valueOf(dbo.get("cubage_percent")) < 9)) { + row << 6.3 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 9 && Double.valueOf(dbo.get("cubage_percent")) < 10)) { + row << 6.5 + } else if (dbo.get("cubage_percent") && Double.valueOf(dbo.get("cubage_percent")) >= 10) { + row << 6.7 + } else { + row << dbo.get("cubage_percent") // 容积率 + } + } else { + def temp = CalcCubageAndVirescence.findResult(dbo.lat / 10000000000.0d, dbo.lng / 10000000000.0d, 500, sameRegionDistricts.get(dbo.region), dbo.name, dbo._id) + if (temp[2] && Double.valueOf(temp[2]) < 1) { + row << Double.valueOf(temp[2]) * 100 + } else if (temp[2] && Double.valueOf(temp[2]) == 1) { + row << Double.valueOf(temp[2]) * 10 + } else if (temp[2] && Double.valueOf(temp[2]) >= 70) { + row << 65 + } else { + row << temp[2] // 绿化率 + } + + if (temp[1] && (Double.valueOf(temp[1]) >= 7 && Double.valueOf(temp[1]) < 8)) { + row << 6.2 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 8 && Double.valueOf(temp[1]) < 9)) { + row << 6.3 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 9 && Double.valueOf(temp[1]) < 10)) { + row << 6.5 + } else if (temp[1] && Double.valueOf(temp[1]) >= 10) { + row << 6.7 + } else { + row << temp[1] // 容积率 + } + } + if (dbo.get("car_site_count") != null) { + if (Long.valueOf(dbo.get("car_site_count")) > 2000 || Long.valueOf(dbo.get("car_site_count")) < 100) { + row << "" + } else { + row << dbo.get("car_site_count") // 车位数量 + } + } else { + row << "" // 车位数量 + } + + row << "" // 介绍 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + map.put(dbo.get("_id").toString(), row) + } catch(Exception e) { + + } finally { + latch.countDown() + } + } + }) + }) + latch.await() + return map + }; + + /** + * 处理特殊字符 + * @param dbo + */ + static def dealSpecialChar(DBObject dbo) { + Set keySet = dbo.keySet() + for (String key : keySet) { + if (String.isInstance(dbo.get(key))) { + dbo.put(key, pattern.matcher(dbo.get(key)).replaceAll("")) + } + } + }; + + static def getSeaweedTypeCode(def value) { + def ret + if (StringUtils.isNotBlank(value)) { + ret = seaweedTypeMap.get(value) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy new file mode 100644 index 000000000..d85ddf81f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +class Temp { + + static main (args){ + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "北京") + + DBObject filter = new BasicDBObject() + filter.append("build_structure", 1) + + Set set = new HashSet<>() + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({ dbo -> + if (dbo.build_structure) { + set.add(dbo.build_structure) + } + }) + + set.each {val -> + println(val) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy new file mode 100644 index 000000000..03d95a32c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 区域工具类 + */ +class DistrictUtils { + + static main(args) { + } + + static def getAllDistricts(def cat, def city, def sameRegionDistricts){ + DBObject query = new BasicDBObject(); + query.append("cat", cat).append("visible", 0).append("city", city) + + DBObject keys = new BasicDBObject().append("city", 1).append("region", 1).append("cubage_percent", 1) + .append("virescence_percent", 1).append("lat2", 1).append("lng2", 1) + .append("finish_building_date", 1).append("name", 1).append("_id", 1); + + DSCursor cursor = DbUtils.seaweed.find(query, keys).batchSize(10000) + Iterator iters = cursor.iterator() + def result = [] + while(iters.hasNext()) { + DBObject obj = iters.next() + + def map = [:]; + map.city = obj.city + map.cubage_percent = obj.cubage_percent; + map.virescence_percent = obj.virescence_percent; + map.lat2 = obj.lat2 + map.lng2 = obj.lng2 + map.region = obj.region + map.finish_building_date = obj.finish_building_date + map.name = obj.name + map._id = obj._id + result << map; + if (sameRegionDistricts.get(obj.region)) { + def temp = sameRegionDistricts.get(obj.region) + temp << map + sameRegionDistricts.put(obj.region, temp) + } else { + def temp = [] + temp << map + sameRegionDistricts.put(obj.region, temp) + } + } + return result; + } +} From b4c7f8ff1a9824291bdd3d20aa2c019f323d233f Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Mon, 17 May 2021 21:15:45 +0800 Subject: [PATCH 045/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=BF=E5=8C=96?= =?UTF-8?q?=E7=8E=87=E3=80=81=E5=AE=B9=E7=A7=AF=E7=8E=87=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E8=AF=9D=E5=B7=A5=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/util/SeaweedUtils.groovy | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 2f93a62e2..22969a8c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.apache.commons.lang.math.NumberUtils import java.util.concurrent.ConcurrentHashMap @@ -29,4 +30,73 @@ class SeaweedUtils { } return region } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && NumberUtils.isDigits(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (virescence_percent == 1) { + return virescence_percent * 10 + } + if (virescence_percent >= 70) { + return 65 + } + return virescence_percent + } + return "" + } + + /** + * 标准化绿化率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && NumberUtils.isDigits(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (cubage_percent && val >= 10) { + return 6.7 + } + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && NumberUtils.isDigits(car_site_count)) { + long val = Long.valueOf("car_site_count") + if (val > 2000 || val < 100) { + return "" + } + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && NumberUtils.isDigits(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } } From 6ec617af10d0b60ca2c5f6756bbcffbbd90e6d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 18 May 2021 10:03:59 +0800 Subject: [PATCH 046/118] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=B1=BB=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 14 ++++ .../script/temp/shenjian/cqrcb/POIRule.groovy | 3 +- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 53 +++--------- .../temp/shenjian/util/SeaweedUtils.groovy | 82 ++++++++++++++++--- 4 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy new file mode 100644 index 000000000..e499d789e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -0,0 +1,14 @@ +package com.fangjia.script.temp.shenjian + +/** + * @author shenjian* @since 2021/5/18 + */ +class Temp { + + static void main(String[] args) { + double val = 1.0 + if (val == 1.0) { + println(true) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index 5deadb989..2c5f787b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -65,6 +65,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -// +//db.getCollection('gaode_address_poi').update({'type': /专科医院/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /诊所/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 66703a1f0..f910ce590 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.temp.shenjian.cqrcb import com.fangjia.core.util.DateUtil import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -86,8 +87,8 @@ class SeaweedExport { DbUtils.seaweed.find(query).limit(1).each { obj -> ret[0] = obj.developer ret[1] = obj.estate_company - ret[2] = obj.cubage_percent - ret[3] = virescencePercentValidator(obj.virescence_percent) + ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) + ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) if (obj.alias) { // 小区可能存在多个别名 @@ -98,22 +99,12 @@ class SeaweedExport { String als = temp.join(",") ret[4] = als.length() > 20 ? "" : als } - ret[5] = obj.car_site_count - if (com.fangjia.script.datastore.FieldValidatorUtils.householdCountValidator(obj.household_count) - && FieldValidatorUtils.carSiteValidator(obj.car_site_count)) { - double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) - if (val == 0.0) { - val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) - } - if (val == 0.0) { - val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) - } - ret[6] = "1:" + val - } + ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) + ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) ret[7] = "" - ret[8] = obj.all_area + ret[8] = SeaweedUtils.formatAllArea(obj.all_area) ret[9] = obj.loudong_count - ret[10] = obj.household_count + ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count) ret[11] = "" ret[12] = obj.estate_charge ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date) @@ -133,37 +124,13 @@ class SeaweedExport { }; latch.await(); def ret = []; - ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"] + ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"]; for (List district : data) { def district_code = district.get(2) def name = district.get(3) ret << resultMap.get(district_code + name) - } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商物业公司.xlsx", ret, "项目统计"); + }; + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商及物业信息.xlsx", ret, "开发商及物业信息"); executorService.shutdown() } - - /** - * string - * 绿化率 仅限数字,包括(小数)数字范围:10-100 - * @param virescence_percent - * @return - */ - private static def virescencePercentValidator(def virescence_percent){ - if(virescence_percent=="null"){ - return ""; - } - def value = com.fangjia.script.datastore.DistrictFieldNumberic.checkValue(virescence_percent); - if(value){ - if(value>=10 && value<=100){ - return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value)) - } else if (value > 0 && value < 1) { - return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value) * 100) - } else{ - return ""; - } - } - return ""; - } - } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 22969a8c8..46d7d1f0d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -1,9 +1,10 @@ package com.fangjia.script.temp.shenjian.util +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import org.apache.commons.lang.math.NumberUtils import java.util.concurrent.ConcurrentHashMap @@ -30,33 +31,33 @@ class SeaweedUtils { } return region } - + /** * 标准化输出绿化率 */ static formatVirescencePercent(def virescence_percent) { // 绿化率 - if (virescence_percent && NumberUtils.isDigits(virescence_percent)) { + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { double val = Double.valueOf(virescence_percent) if (val < 1) { return val * 100 } - if (virescence_percent == 1) { - return virescence_percent * 10 + if (val == 1) { + return val * 10 } - if (virescence_percent >= 70) { + if (val >= 70) { return 65 } - return virescence_percent + return val } return "" } /** - * 标准化绿化率输出 + * 标准化容积率输出 */ static formatCubagePercent(def cubage_percent) { - if (cubage_percent && NumberUtils.isDigits(cubage_percent)) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { double val = Double.valueOf(cubage_percent) if (val >= 7 && val < 8) { return 6.2 @@ -67,9 +68,10 @@ class SeaweedUtils { if (val >= 9 && val < 10) { return 6.5 } - if (cubage_percent && val >= 10) { + if (val >= 10) { return 6.7 } + return val } return "" } @@ -78,11 +80,12 @@ class SeaweedUtils { * 标准化车位输出 */ static formatCarSiteCount(def car_site_count) { - if (car_site_count && NumberUtils.isDigits(car_site_count)) { - long val = Long.valueOf("car_site_count") + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) if (val > 2000 || val < 100) { return "" } + return val; } return "" } @@ -91,7 +94,7 @@ class SeaweedUtils { * 标准化建筑面积输出 */ static formatBuildArea(def build_area) { - if (build_area && NumberUtils.isDigits(build_area)) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { double val = build_area if (val > 1000) { return val @@ -99,4 +102,57 @@ class SeaweedUtils { } return "" } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } } From ae5041e93f1f33e8e4ed8064bf8217a712d3051c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 18 May 2021 17:09:23 +0800 Subject: [PATCH 047/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9?= =?UTF-8?q?=E9=85=8D=E5=A5=97=E6=8E=92=E5=BA=8F=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 21 +++++++++------- .../script/temp/shenjian/cqrcb/POIRule.groovy | 4 --- .../script/temp/shenjian/dto/AroundDTO.groovy | 25 +++++++++++++++++++ 3 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 4ec4dac97..36e9e588f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.district.Distance import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -96,10 +97,13 @@ class AroundExportByNewWay { Integer key = 1; def ret = [] for (List district : data) { + def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { - ret.addAll(resultMap.get(district_code + name)) + List currRet = resultMap.get(district_code + name) + Collections.sort(currRet) + ret.addAll(currRet) if (ret.size() > 600000) { map.put(key++, ret) ret = [] @@ -110,11 +114,10 @@ class AroundExportByNewWay { //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); for (Integer k : map.keySet()) { //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") - EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套(文件"+ k + ")V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); } printUnMatched(unMatchedMap) executorService.shutdown() - } static calcAroundInfo(def lat, def lng, def result, def id) { @@ -130,12 +133,12 @@ class AroundExportByNewWay { if (distance == 0) { distance = 50 } - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = formatType(obj.typecode) - ret[3] = distance.toInteger() - result << ret + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index 2c5f787b2..1c907e236 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -46,17 +46,13 @@ class POIRule { // 更新规则 // db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDEFHG]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /购物服务;商场;免税品店/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;口腔医院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;综合医院;卫生院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[学]+楼/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[教医光]+学中心/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;学校/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[1234]期/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /住院部/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy new file mode 100644 index 000000000..39ed00ec4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.dto + +/** + * @author shenjian + * @since 2021/5/18 + */ +class AroundDTO implements Comparable { + + String id; + String name; + String type + Integer distance + + @Override + public int compareTo(AroundDTO o2) { + if (this.type.equals(o2.type)) { + if (this.distance != null) { + return this.distance.compareTo(o2.distance); + } + } else { + return this.type.compareTo(o2.type); + } + return 0; + } +} From e622b12ca788f490c25e10ef521428226fdaff4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 18 May 2021 19:49:59 +0800 Subject: [PATCH 048/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 1 + .../cqrcb/AroundExportByNewWay.groovy | 38 ++++++++++++------- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 11 +----- .../temp/shenjian/util/SeaweedUtils.groovy | 25 ++++++++++++ 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e499d789e..e8a43efdb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -10,5 +10,6 @@ class Temp { if (val == 1.0) { println(true) } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 36e9e588f..5d676e7c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -96,11 +96,13 @@ class AroundExportByNewWay { Map map = new HashedMap() Integer key = 1; def ret = [] + AtomicInteger totalCount = new AtomicInteger() for (List district : data) { def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() List currRet = resultMap.get(district_code + name) Collections.sort(currRet) ret.addAll(currRet) @@ -116,6 +118,7 @@ class AroundExportByNewWay { //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") EasyExcel.write("F:\\数据\\重庆农商行\\周边配套(文件"+ k + ")V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); } + print(totalCount.get() + " " + (totalCount.get() / data.size())) printUnMatched(unMatchedMap) executorService.shutdown() } @@ -154,19 +157,28 @@ class AroundExportByNewWay { com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); } - static String formatType(String type) { - if (type.startsWith("09")) { - return "医院"; - } else if (type.startsWith('1505')) { - return "地铁"; - } else if (type.startsWith('1507')) { - return "公交"; - } else if (type.startsWith("16")) { - return "银行"; - } else if (type.startsWith("14")) { - return "学校"; - } else if (type.startsWith("06")) { - return "商场" + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index f910ce590..793ff7840 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -89,16 +89,7 @@ class SeaweedExport { ret[1] = obj.estate_company ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) - - if (obj.alias) { - // 小区可能存在多个别名 - def temp = [] - for (def a : obj.alias) { - temp.add(a) - } - String als = temp.join(",") - ret[4] = als.length() > 20 ? "" : als - } + ret[4] = SeaweedUtils.formatAlias(obj.alias) ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) ret[7] = "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 46d7d1f0d..2cb9a61b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -155,4 +155,29 @@ class SeaweedUtils { } return "" } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } } From f9f80f733eadee29d692895ff5ccbdab6cacfa52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 19 May 2021 11:46:43 +0800 Subject: [PATCH 049/118] =?UTF-8?q?=E5=85=AC=E4=BA=A4=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 23 +- .../cqrcb/AroundExportByNewWay.groovy | 23 +- .../shenjian/cqrcb/AroundFormatExport.groovy | 231 ++++++++++-------- .../cqrcb/CheckLatLngByDistrict.groovy | 61 +++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 8 +- 6 files changed, 233 insertions(+), 117 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e8a43efdb..8c42ab478 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,15 +1,30 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.lixue.util.DBUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - double val = 1.0 - if (val == 1.0) { - println(true) + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("region", "北碚") + query.put("cat", "district") + query.put("visible", 0) + query.put("name", "人民村") + DBUtils.seaweed.find(query).limit(1).each { obj -> + println(obj) } - } + +// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 +// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 +// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米 +// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 5d676e7c9..3bffc1300 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -59,11 +59,12 @@ class AroundExportByNewWay { query.put("cat", "district") query.put("visible", 0) query.put("name", district.get(24)) + Set set = new HashSet<>() DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> if (obj.lng && obj.lat) { def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) - calcAroundInfo(lat, lng, result, id) + calcAroundInfo(lat, lng, result, id, set) } } } else { @@ -106,7 +107,7 @@ class AroundExportByNewWay { List currRet = resultMap.get(district_code + name) Collections.sort(currRet) ret.addAll(currRet) - if (ret.size() > 600000) { + if (ret.size() > 700000) { map.put(key++, ret) ret = [] } @@ -123,7 +124,7 @@ class AroundExportByNewWay { executorService.shutdown() } - static calcAroundInfo(def lat, def lng, def result, def id) { + static calcAroundInfo(def lat, def lng, def result, def id, def set) { // 周围半径为3千米 def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) DBObject q = new BasicDBObject() @@ -136,12 +137,16 @@ class AroundExportByNewWay { if (distance == 0) { distance = 50 } - AroundDTO aroundDTO = new AroundDTO() - aroundDTO.id = id - aroundDTO.name = obj.name - aroundDTO.type = formatType(obj.typecode) - aroundDTO.distance = distance.toInteger() - result.add(aroundDTO) + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy index b445e881c..9a3f67a3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -1,31 +1,36 @@ package com.fangjia.script.temp.shenjian.cqrcb -import cn.emay.mina.util.ConcurrentHashSet -import com.fangjia.script.coredata.util.CollectionUtils -import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.alibaba.excel.EasyExcel +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger /** - * 小区周围信息格式化输出 + * 周边数据一千米配套数量统计 * * @author shenjian - * @since 2021/5/15 + * @since 2021/5/16 */ class AroundFormatExport { - static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -38,124 +43,148 @@ class AroundFormatExport { @Override void run() { try { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - def temp = [] - if (district.get(20) != null && district.get(21) != null) { - def matched_city = district.get(22) - def matched_region = district.get(23) - def matched_name = district.get(24) + List result = [] + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { DBObject query = new BasicDBObject() - query.put("matched_city", matched_city) - query.put("matched_region", matched_region) - query.put("matched_name", matched_name) - List ret = new ArrayList() - Map map = new ConcurrentHashMap<>(); - conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> - dealSpecial(obj, map, ret) + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + Set set = new HashSet<>() + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id, set) + } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - ret.add(key + ":" + (distance / count1) + "米") - } - - temp[7] = matched_city - temp[8] = matched_region - temp[9] = matched_name - if (ret.size() > 0) { - def str = ret.join(",") - temp[5] = str - println(str) - } - + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } - resultMap.put(district_code + name, temp) + resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) } catch (Exception ex) { - + ex.printStackTrace() } finally { latch.countDown() } } }) + }; + latch.await(); + def headArr = ['IDB', '数量'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) } - latch.await() + Map map = new HashedMap() + Integer key = 1; def ret = [] - ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] + AtomicInteger totalCount = new AtomicInteger() for (List district : data) { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - ret << resultMap.get(district_code + name) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() + List currRet = resultMap.get(district_code + name) + ret << [id, currRet.size()] + if (ret.size() > 700000) { + map.put(key++, ret) + ret = [] + } + } else { + ret << [id, 0] + } } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + map.put(key++, ret) + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + for (Integer k : map.keySet()) { + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + print(totalCount.get() + " " + (totalCount.get() / data.size())) + printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def ret) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false - } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false + static calcAroundInfo(def lat, def lng, def result, def id, def set) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 1000) { + if (distance == 0) { + distance = 50 + } + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) } } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - int distance = Double.valueOf(obj.distance) + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - map.put(key, child) - } else { - child.put("distance", Double.valueOf(obj.distance)) - child.put("count", 1) - map.put(key, child) - } - } else { - ret.add(obj.name + ":" + obj.distance + "米") } } + + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] + } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy new file mode 100644 index 000000000..a5338340b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger + +/** + * 通过匹配出的小区核对经纬度 + * + * @author shenjian + * @since 2021/5/19 + */ +class CheckLatLngByDistrict { + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + for (List districtT : data) { + final List district = districtT + if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + if (district.get(20) != null && district.get(21) != null) { + def originLng = district.get(20) + def originLat = district.get(21) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + if (originLng != obj.lng || originLat != obj.lat) { + def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d); + def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d); + originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d); + originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d); + + double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng) + println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" ) + } + } else { + println(district.get(24) + " 不存在经纬度") + } + } + } else { + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + + } else { + println(district.get(24) + " 不存在经纬度222222222") + } + } + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 793ff7840..86504ae9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -85,8 +85,8 @@ class SeaweedExport { filter.put("lat", 1) DbUtils.seaweed.find(query).limit(1).each { obj -> - ret[0] = obj.developer - ret[1] = obj.estate_company + ret[0] = obj.estate_company + ret[1] = obj.developer ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) ret[4] = SeaweedUtils.formatAlias(obj.alias) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 3e745dfd3..ce6475a95 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -45,7 +45,7 @@ class GaoDeUtils { }}; /** - * + * 查找周边配套 * @param city * @param kw * @param lng @@ -87,10 +87,16 @@ class GaoDeUtils { return result } + /** + * 获取经纬度之间距离 + */ static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) } + /** + * 获取地址经纬度 + */ static def parserByGaode(def city, def address) { JSONObject json = GaoDeApiUtils.geo(city, address) if (json && json.status == "1") { From 7726fa48cb6680a639242015d64c9b6a352296ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 19 May 2021 11:46:43 +0800 Subject: [PATCH 050/118] =?UTF-8?q?=E5=85=AC=E4=BA=A4=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 23 +- .../cqrcb/AroundExportByNewWay.groovy | 23 +- .../shenjian/cqrcb/AroundFormatExport.groovy | 252 +++++++++++------- .../cqrcb/CheckLatLngByDistrict.groovy | 61 +++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 8 +- 6 files changed, 254 insertions(+), 117 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e8a43efdb..8c42ab478 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,15 +1,30 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.lixue.util.DBUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - double val = 1.0 - if (val == 1.0) { - println(true) + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("region", "北碚") + query.put("cat", "district") + query.put("visible", 0) + query.put("name", "人民村") + DBUtils.seaweed.find(query).limit(1).each { obj -> + println(obj) } - } + +// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 +// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 +// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米 +// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 5d676e7c9..3bffc1300 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -59,11 +59,12 @@ class AroundExportByNewWay { query.put("cat", "district") query.put("visible", 0) query.put("name", district.get(24)) + Set set = new HashSet<>() DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> if (obj.lng && obj.lat) { def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) - calcAroundInfo(lat, lng, result, id) + calcAroundInfo(lat, lng, result, id, set) } } } else { @@ -106,7 +107,7 @@ class AroundExportByNewWay { List currRet = resultMap.get(district_code + name) Collections.sort(currRet) ret.addAll(currRet) - if (ret.size() > 600000) { + if (ret.size() > 700000) { map.put(key++, ret) ret = [] } @@ -123,7 +124,7 @@ class AroundExportByNewWay { executorService.shutdown() } - static calcAroundInfo(def lat, def lng, def result, def id) { + static calcAroundInfo(def lat, def lng, def result, def id, def set) { // 周围半径为3千米 def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) DBObject q = new BasicDBObject() @@ -136,12 +137,16 @@ class AroundExportByNewWay { if (distance == 0) { distance = 50 } - AroundDTO aroundDTO = new AroundDTO() - aroundDTO.id = id - aroundDTO.name = obj.name - aroundDTO.type = formatType(obj.typecode) - aroundDTO.distance = distance.toInteger() - result.add(aroundDTO) + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy index b445e881c..89b1c7b3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -1,31 +1,36 @@ package com.fangjia.script.temp.shenjian.cqrcb -import cn.emay.mina.util.ConcurrentHashSet -import com.fangjia.script.coredata.util.CollectionUtils -import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.alibaba.excel.EasyExcel +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger /** - * 小区周围信息格式化输出 + * 周边数据一千米配套数量统计 * * @author shenjian - * @since 2021/5/15 + * @since 2021/5/16 */ class AroundFormatExport { - static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -38,124 +43,169 @@ class AroundFormatExport { @Override void run() { try { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - def temp = [] - if (district.get(20) != null && district.get(21) != null) { - def matched_city = district.get(22) - def matched_region = district.get(23) - def matched_name = district.get(24) + List result = [] + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { DBObject query = new BasicDBObject() - query.put("matched_city", matched_city) - query.put("matched_region", matched_region) - query.put("matched_name", matched_name) - List ret = new ArrayList() - Map map = new ConcurrentHashMap<>(); - conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> - dealSpecial(obj, map, ret) + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + Set set = new HashSet<>() + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id, set) + } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - ret.add(key + ":" + (distance / count1) + "米") - } - - temp[7] = matched_city - temp[8] = matched_region - temp[9] = matched_name - if (ret.size() > 0) { - def str = ret.join(",") - temp[5] = str - println(str) - } - + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } - resultMap.put(district_code + name, temp) + resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) } catch (Exception ex) { - + ex.printStackTrace() } finally { latch.countDown() } } }) + }; + latch.await(); + def headArr = ['IDB', '数量'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) } - latch.await() + Map map = new HashedMap() + Integer key = 1; def ret = [] - ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] + AtomicInteger totalCount = new AtomicInteger() for (List district : data) { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - ret << resultMap.get(district_code + name) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() + List currRet = resultMap.get(district_code + name) + int gj = 0 + int dt = 0 + int yh = 0 + int yy = 0 + int sc = 0 + int xx = 0 + for (AroundDTO dto : currRet) { + if(dto.getType() == '公交') { + gj++ + } else if(dto.getType() == '地铁') { + dt++ + } else if(dto.getType() == '银行') { + yh++ + } else if (dto.getType() == '医院') { + yy++ + } else if(dto.getType() == '商场') { + sc++ + } else if(dto.getType() == '学校') { + xx++ + } + } + ret << [id, "公交:" + gj + ";" + "地铁:" + dt + ";银行:" + yh + ";医院:" + yy + ";商场:" + sc + ";学校:" + xx] + if (ret.size() > 700000) { + map.put(key++, ret) + ret = [] + } + } else { + ret << [id, ""] + } } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + map.put(key++, ret) + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + for (Integer k : map.keySet()) { + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + print(totalCount.get() + " " + (totalCount.get() / data.size())) + printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def ret) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false - } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false + static calcAroundInfo(def lat, def lng, def result, def id, def set) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 1000) { + if (distance == 0) { + distance = 50 + } + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) } } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - int distance = Double.valueOf(obj.distance) + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - map.put(key, child) - } else { - child.put("distance", Double.valueOf(obj.distance)) - child.put("count", 1) - map.put(key, child) - } - } else { - ret.add(obj.name + ":" + obj.distance + "米") } } + + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] + } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy new file mode 100644 index 000000000..a5338340b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger + +/** + * 通过匹配出的小区核对经纬度 + * + * @author shenjian + * @since 2021/5/19 + */ +class CheckLatLngByDistrict { + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + for (List districtT : data) { + final List district = districtT + if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + if (district.get(20) != null && district.get(21) != null) { + def originLng = district.get(20) + def originLat = district.get(21) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + if (originLng != obj.lng || originLat != obj.lat) { + def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d); + def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d); + originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d); + originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d); + + double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng) + println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" ) + } + } else { + println(district.get(24) + " 不存在经纬度") + } + } + } else { + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + + } else { + println(district.get(24) + " 不存在经纬度222222222") + } + } + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 793ff7840..86504ae9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -85,8 +85,8 @@ class SeaweedExport { filter.put("lat", 1) DbUtils.seaweed.find(query).limit(1).each { obj -> - ret[0] = obj.developer - ret[1] = obj.estate_company + ret[0] = obj.estate_company + ret[1] = obj.developer ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) ret[4] = SeaweedUtils.formatAlias(obj.alias) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 3e745dfd3..ce6475a95 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -45,7 +45,7 @@ class GaoDeUtils { }}; /** - * + * 查找周边配套 * @param city * @param kw * @param lng @@ -87,10 +87,16 @@ class GaoDeUtils { return result } + /** + * 获取经纬度之间距离 + */ static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) } + /** + * 获取地址经纬度 + */ static def parserByGaode(def city, def address) { JSONObject json = GaoDeApiUtils.geo(city, address) if (json && json.status == "1") { From a00200c12e78977bf322cb4e3a9ab9794c6f012b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 11:22:13 +0800 Subject: [PATCH 051/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9POI?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AllExport.groovy | 2 +- .../temp/shenjian/cqrcb/AroundExport.groovy | 2 +- .../script/temp/shenjian/cqrcb/SavePoi.groovy | 2 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 124 ++++++++++++++++++ .../fangjia/script/temp/shenjian/poi/poi.json | 38 ++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 70 +++++++++- .../temp/shenjian/util/HttpUtils.groovy | 29 ++++ .../temp/shenjian/util/SeaweedUtils.groovy | 8 ++ .../wenyan/auto/KeywordSearchUtils.groovy | 1 + .../script/statistics/poi/AmapPoi.java | 71 ++++++++++ 10 files changed, 338 insertions(+), 9 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy index 99bf24d62..0ed85bb0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy @@ -27,7 +27,7 @@ class AllExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy index fd3450f87..28cec6534 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -27,7 +27,7 @@ class AroundExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 //static ExecutorService executorService = Executors.newFixedThreadPool(30) // 高德地图中不存在周边的数据 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy index 9da03d468..40c42f36a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -21,7 +21,7 @@ class SavePoi { static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 static ExecutorService executorService = Executors.newFixedThreadPool(30) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..9b2599516 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoi { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(5000).each {obj -> + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json new file mode 100644 index 000000000..4777a3761 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json @@ -0,0 +1,38 @@ +{ + "id": "B0FFF9RJ2O", + "parent": [], + "childtype": [], + "name": "丁兰广场", + "type": "购物服务;商场;购物中心", + "typecode": "060101", + "biz_type": [], + "address": "惠兰雅路与明珠街交汇处东北角", + "location": "120.231368,30.353127", + "tel": "0571-88111223", + "pname": "浙江省", + "cityname": "杭州市", + "adname": "上城区", + "importance": [], + "shopid": [], + "shopinfo": "0", + "poiweight": [], + "distance": "37", + "biz_ext": { + "rating": [], + "cost": "13953.00" + }, + "photos": [ + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/f5dcc118938358031b45dea7e04faf45" + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/9a0fd6ceec63e030eac3ee14055206b5" + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/d9d2a87de3f00294665a2caeb3ee226f" + } + ] +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index ce6475a95..f772c3ab8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,12 +1,12 @@ package com.fangjia.script.temp.shenjian.util +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import net.sf.json.JSONObject - -import java.util.concurrent.ConcurrentHashMap /** * 高德工具类 @@ -16,7 +16,8 @@ import java.util.concurrent.ConcurrentHashMap */ class GaoDeUtils { - def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988" + // 新获取key + def static GAO_DE_AK = "4e9592208336433ee1b7aa4d1d434418" def static codeNameMap = new HashMap(){{ put("150500", "地铁"); @@ -54,7 +55,7 @@ class GaoDeUtils { * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) * @return */ - static List searchNearByPoi(String city, String types, Double lng, Double lat, String radius, String keywords){ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] params.key = GAO_DE_AK @@ -64,7 +65,7 @@ class GaoDeUtils { params.keywords = keywords } params.city = city - params.radius = radius + params.radius = String.valueOf(radius) params.location = "${lng},${lat}".toString() List result = [] @@ -80,6 +81,63 @@ class GaoDeUtils { if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + JSONArray jsonArray = json.pois + List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); + result.addAll(ret) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; } else { break } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 2cb9a61b2..22673723b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -176,6 +176,14 @@ class SeaweedUtils { return "" } + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + static void main(String[] args) { def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] println(formatAlias(alias)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy index a2026322b..c3a0c4acf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy @@ -204,6 +204,7 @@ public class KeywordSearchUtils{ put("旅游景点","110000"); put("体育场馆","0800|0801"); put("图书馆","140500"); + put("住宅区","120300|120301|120302|120303|120304"); }}; diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From d81f24748874c32ea2c9d4682dc1a99ed8a563a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 14:06:16 +0800 Subject: [PATCH 052/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BF=BB=E9=A1=B5?= =?UTF-8?q?=E5=90=8E=E9=95=BF=E5=BA=A6=E4=B8=BA1=E8=BD=ACjsonArray?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 16 ++-- .../fangjia/script/temp/shenjian/poi/poi.json | 84 +++++++++++-------- .../temp/shenjian/util/GaoDeUtils.groovy | 9 +- 3 files changed, 60 insertions(+), 49 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 8c42ab478..8311e3e15 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,6 +1,9 @@ package com.fangjia.script.temp.shenjian +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi import com.mongodb.BasicDBObject import com.mongodb.DBObject @@ -10,15 +13,10 @@ import com.mongodb.DBObject class Temp { static void main(String[] args) { - DBObject query = new BasicDBObject() - query.put("city", "重庆") - query.put("region", "北碚") - query.put("cat", "district") - query.put("visible", 0) - query.put("name", "人民村") - DBUtils.seaweed.find(query).limit(1).each { obj -> - println(obj) - } + String jsonStr = "[{\"id\":\"B0FFLCY016\",\"parent\":[],\"childtype\":[],\"name\":\"杭州馨悦坊公寓\",\"type\":\"住宿服务;住宿服务相关;住宿服务相关\",\"typecode\":\"100000\",\"biz_type\":\"hotel\",\"address\":\"18号大街伊萨卡浩泽园\",\"location\":\"120.383947,30.289931\",\"tel\":[],\"pname\":\"浙江省\",\"cityname\":\"杭州市\",\"adname\":\"钱塘区\",\"importance\":[],\"shopid\":[],\"shopinfo\":\"0\",\"poiweight\":[],\"distance\":\"992\",\"biz_ext\":{\"rating\":[],\"cost\":[],\"hotel_ordering\":\"0\",\"lowest_price\":[],\"star\":[],\"open_time\":[],\"opentime2\":[]},\"photos\":[{\"title\":\"Logo\",\"url\":\"http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c\",\"provider\":[]},{\"title\":\"客房\",\"url\":\"http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74\",\"provider\":[]},{\"title\":[],\"url\":\"http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315\",\"provider\":[]}]}]" + JSONArray jsonArray = JSONObject.parseArray(jsonStr) + List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); + print(ret.size()) } // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json index 4777a3761..2cc6544cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json @@ -1,38 +1,48 @@ -{ - "id": "B0FFF9RJ2O", - "parent": [], - "childtype": [], - "name": "丁兰广场", - "type": "购物服务;商场;购物中心", - "typecode": "060101", - "biz_type": [], - "address": "惠兰雅路与明珠街交汇处东北角", - "location": "120.231368,30.353127", - "tel": "0571-88111223", - "pname": "浙江省", - "cityname": "杭州市", - "adname": "上城区", - "importance": [], - "shopid": [], - "shopinfo": "0", - "poiweight": [], - "distance": "37", - "biz_ext": { - "rating": [], - "cost": "13953.00" - }, - "photos": [ - { - "title": [], - "url": "http://store.is.autonavi.com/showpic/f5dcc118938358031b45dea7e04faf45" +[ + { + "id": "B0FFLCY016", + "parent": [], + "childtype": [], + "name": "杭州馨悦坊公寓", + "type": "住宿服务;住宿服务相关;住宿服务相关", + "typecode": "100000", + "biz_type": "hotel", + "address": "18号大街伊萨卡浩泽园", + "location": "120.383947,30.289931", + "tel": [], + "pname": "浙江省", + "cityname": "杭州市", + "adname": "钱塘区", + "importance": [], + "shopid": [], + "shopinfo": "0", + "poiweight": [], + "distance": "992", + "biz_ext": { + "rating": [], + "cost": [], + "hotel_ordering": "0", + "lowest_price": [], + "star": [], + "open_time": [], + "opentime2": [] }, - { - "title": [], - "url": "http://store.is.autonavi.com/showpic/9a0fd6ceec63e030eac3ee14055206b5" - }, - { - "title": [], - "url": "http://store.is.autonavi.com/showpic/d9d2a87de3f00294665a2caeb3ee226f" - } - ] -} \ No newline at end of file + "photos": [ + { + "title": "Logo", + "url": "http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c", + "provider": [] + }, + { + "title": "客房", + "url": "http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74", + "provider": [] + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315", + "provider": [] + } + ] + } +] \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index f772c3ab8..9f277a4b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -129,9 +129,12 @@ class GaoDeUtils { } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() - JSONArray jsonArray = json.pois - List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); - result.addAll(ret) + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } } else if (json && json.infocode && json.infocode == 10003) { throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") } else if (json && json.infocode && json.infocode == 10004) { From 9f5d0ca222cf4ff53f92862fc3849acb97a3e09d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 14:19:29 +0800 Subject: [PATCH 053/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=A4=9A=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=96=B9=E5=BC=8F=E8=B0=83=E7=94=A8=E5=91=A8=E8=BE=B9?= =?UTF-8?q?POI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 37 ++++++++++++------- .../temp/shenjian/util/GaoDeUtils.groovy | 8 ++-- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 9b2599516..28ac4a0ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -16,6 +16,9 @@ import com.mongodb.DBObject import org.apache.commons.collections.CollectionUtils import org.bson.types.ObjectId +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + /** * 小区周边POI数据获取 * @@ -26,26 +29,34 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { def types = listAllType() //def types = "0601" DBObject query = new BasicDBObject() query.put("status", 1) - conn136.find(query).batchSize(5000).each {obj -> - def city = obj.city - def lng = obj.lng - def lat = obj.lat - List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") - if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) - HttpUtils.postForJson(URL, jsonStr) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) - } else { - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) - } - println("已处理小区:" + obj) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) } + executorService.shutdown() } static getNewSeaweed() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 9f277a4b2..e384c35fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -135,9 +135,11 @@ class GaoDeUtils { } catch(Exception ex) { ex.printStackTrace() } - } else if (json && json.infocode && json.infocode == 10003) { - throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") - } else if (json && json.infocode && json.infocode == 10004) { + } else if (json && json.infocode && json.infocode == "10044") { + //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + // 多线程情况下还是exit有效果,抛异常不会停止程序 + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; From bf14e84e49198979efecc3bbb6b884d866d5866d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 15:52:01 +0800 Subject: [PATCH 054/118] =?UTF-8?q?=E6=9A=82=E6=97=B6=E5=B1=8F=E8=94=BD?= =?UTF-8?q?=E4=B8=8D=E9=87=8D=E8=A6=81=E7=B1=BB=E5=9E=8B=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../wenyan/auto/KeywordSearchUtils.groovy | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy index c3a0c4acf..278aced74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy @@ -179,32 +179,33 @@ public class KeywordSearchUtils{ public static final Map KEYWORDS_MAP = new HashMap(){{ put("地铁", "150500"); put("公交", "1507"); - put("码头", "1503"); - put("机场", "150104|150106"); - put("火车站", "150200"); put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); put("写字楼", "120201"); - put("停车场", "150903|150904|150905|150906"); - put("网吧", "080308"); put("学校","141200|141201|141202|141203|141204"); put("银行","1601|1603"); - put("公园/绿地","110100|110101"); - put("KTV","080302"); - put("健身中心","080111"); put("大型超市","0604"); - put("咖啡厅","0505"); - put("剧场/影院","080601|080603"); put("商场","0601"); - put("酒店","10"); - put("整容整形","090201"); - put("洗浴中兴","0714"); - put("便利店","0602"); put("餐饮","0501|0502|0503|0504"); - put("游乐场","080501"); put("旅游景点","110000"); - put("体育场馆","0800|0801"); put("图书馆","140500"); - put("住宅区","120300|120301|120302|120303|120304"); + +// put("码头", "1503");1 +// put("机场", "150104|150106");1 +// put("火车站", "150200");1 +// put("停车场", "150903|150904|150905|150906");1 +// put("网吧", "080308");1 +// put("公园/绿地","110100|110101"); +// put("KTV","080302");1 +// put("健身中心","080111");1 +// put("咖啡厅","0505");1 +// put("剧场/影院","080601|080603");1 +// put("酒店","10");1 +// put("整容整形","090201");1 +// put("洗浴中兴","0714");1 +// put("便利店","0602");1 +// put("游乐场","080501");1 +// put("体育场馆","0800|0801");1 +// put("住宅区","120300|120301|120302|120303|120304"); }}; From 387f6ec98b8f0cbe362afbde21c5d88d140bddb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 18:01:23 +0800 Subject: [PATCH 055/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BAgeo?= =?UTF-8?q?hash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 14 +++--- .../seaweed/SeaweedGeoHashExport.groovy | 44 +++++++++++++++++++ .../script/temp/shenjian/seaweed/Temp.groovy | 28 ------------ .../temp/shenjian/util/SeaweedUtils.groovy | 17 +++++++ 4 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 8311e3e15..e95cbadff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -2,10 +2,9 @@ package com.fangjia.script.temp.shenjian import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject -import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.statistics.poi.AmapPoi -import com.mongodb.BasicDBObject -import com.mongodb.DBObject +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import org.elasticsearch.geometry.utils.Geohash /** * @author shenjian* @since 2021/5/18 @@ -13,10 +12,11 @@ import com.mongodb.DBObject class Temp { static void main(String[] args) { - String jsonStr = "[{\"id\":\"B0FFLCY016\",\"parent\":[],\"childtype\":[],\"name\":\"杭州馨悦坊公寓\",\"type\":\"住宿服务;住宿服务相关;住宿服务相关\",\"typecode\":\"100000\",\"biz_type\":\"hotel\",\"address\":\"18号大街伊萨卡浩泽园\",\"location\":\"120.383947,30.289931\",\"tel\":[],\"pname\":\"浙江省\",\"cityname\":\"杭州市\",\"adname\":\"钱塘区\",\"importance\":[],\"shopid\":[],\"shopinfo\":\"0\",\"poiweight\":[],\"distance\":\"992\",\"biz_ext\":{\"rating\":[],\"cost\":[],\"hotel_ordering\":\"0\",\"lowest_price\":[],\"star\":[],\"open_time\":[],\"opentime2\":[]},\"photos\":[{\"title\":\"Logo\",\"url\":\"http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c\",\"provider\":[]},{\"title\":\"客房\",\"url\":\"http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74\",\"provider\":[]},{\"title\":[],\"url\":\"http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315\",\"provider\":[]}]}]" - JSONArray jsonArray = JSONObject.parseArray(jsonStr) - List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); - print(ret.size()) + String str = SeaweedUtils.encodeByGeoHash(106.569161, 29.498502) + println(str) + str = Geohash.stringEncode(106.569161, 29.498502, 10) + println(str) + // wm5zc4131rmf } // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy new file mode 100644 index 000000000..dad81abe4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区geohash导出 + * + * @author shenjian + * @since 2021/5/25 + */ +class SeaweedGeoHashExport { + + static void main(String[] args) { + Set set = new HashSet<>() + + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("lng", 1) + filter.put("lat", 1) + DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp -> + final def obj = objTemp + if (obj.lng && obj.lat) { + def lon = obj.lng + def lat = obj.lat + set.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) + } + } + def ret = [] + ret << ['小区geohash'] + for (String val : set) { + ret << [val] + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy deleted file mode 100644 index d85ddf81f..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy +++ /dev/null @@ -1,28 +0,0 @@ -package com.fangjia.script.temp.shenjian.seaweed - -import com.fangjia.script.util.DbUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBObject - -class Temp { - - static main (args){ - DBObject query = new BasicDBObject() - query.append("cat", "district").append("visible", 0).append("city", "北京") - - DBObject filter = new BasicDBObject() - filter.append("build_structure", 1) - - Set set = new HashSet<>() - - DbUtils.seaweed.find(query, filter).batchSize(10000).each({ dbo -> - if (dbo.build_structure) { - set.add(dbo.build_structure) - } - }) - - set.each {val -> - println(val) - } - } -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 22673723b..9fa4739f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.datastore.FieldValidatorUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash import java.util.concurrent.ConcurrentHashMap @@ -184,6 +185,22 @@ class SeaweedUtils { return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) } + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + static void main(String[] args) { def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] println(formatAlias(alias)) From 0aa41d570a5b31f0450ae3a0f45c776f3ae1a3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 18:15:35 +0800 Subject: [PATCH 056/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BAgeo?= =?UTF-8?q?hash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/SeaweedGeoHashExport.groovy | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy index dad81abe4..154efe43e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy @@ -5,10 +5,6 @@ import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject -import java.util.concurrent.CountDownLatch -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors - /** * 小区geohash导出 * @@ -18,10 +14,10 @@ import java.util.concurrent.Executors class SeaweedGeoHashExport { static void main(String[] args) { - Set set = new HashSet<>() + List list = new ArrayList<>() DBObject query = new BasicDBObject() - query.append("cat", "district").append("visible", 0) + query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁") DBObject filter = new BasicDBObject() filter.put("lng", 1) @@ -31,12 +27,12 @@ class SeaweedGeoHashExport { if (obj.lng && obj.lat) { def lon = obj.lng def lat = obj.lat - set.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) + list.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) } } def ret = [] ret << ['小区geohash'] - for (String val : set) { + for (String val : list) { ret << [val] } com.fangjia.script.util.ExcelUtils.write("F:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); From 5ccb7e5e212d24639996e4510cba707b0d4eee3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 11:57:30 +0800 Subject: [PATCH 057/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=80=BB=E9=87=8F=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/{ => 58}/City58.groovy | 0 .../shenjian/crawl/{ => 58}/Comm58.groovy | 0 .../shenjian/crawl/58/Count58Seaweed.groovy | 49 +++++++ .../shenjian/crawl/{ => 58}/Seed58.groovy | 0 .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 2 +- .../poi/SeaweedAroundPoiSingal.groovy | 128 ++++++++++++++++++ 6 files changed, 178 insertions(+), 1 deletion(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/City58.groovy (100%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Comm58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Seed58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy new file mode 100644 index 000000000..0c9ea6f30 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.crawl.InvokeHttpRequest +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * 统计58小区总量 + * + * @author shenjian + * @since 2021/5/26 + */ +class Count58Seaweed { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { obj -> + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".58.com/xiaoqu/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + String str = JQueryUtils.selectText(".total-info", doc) + // 117.60.236.41 + if (str) { + str = str.replace("共找到", "").replace("个小区", "").trim() + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count))) + println("当前处理城市" + obj.city) + } + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 28ac4a0ab..016d7be75 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,7 +29,7 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { def types = listAllType() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy new file mode 100644 index 000000000..be70b79e5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -0,0 +1,128 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取单线程方式 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiSingal { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(30).each {objTemp -> + final def obj = objTemp + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} From bc741734477f00b27196a8bd56c212d0fafc04d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 11:57:30 +0800 Subject: [PATCH 058/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=80=BB=E9=87=8F=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/{ => 58}/City58.groovy | 0 .../shenjian/crawl/{ => 58}/Comm58.groovy | 0 .../shenjian/crawl/58/Count58Seaweed.groovy | 65 +++++++++ .../shenjian/crawl/{ => 58}/Seed58.groovy | 0 .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 2 +- .../poi/SeaweedAroundPoiSingal.groovy | 128 ++++++++++++++++++ 6 files changed, 194 insertions(+), 1 deletion(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/City58.groovy (100%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Comm58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Seed58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy new file mode 100644 index 000000000..ca8e4379c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.crawl.InvokeHttpRequest +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * 统计58小区总量 + * + * @author shenjian + * @since 2021/5/26 + */ +class Count58Seaweed { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + crawlSeaweedCount() + println(count()) + } + + static crawlSeaweedCount() { + cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { obj -> + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".58.com/xiaoqu/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + String str = JQueryUtils.selectText(".total-info", doc) + // 117.60.236.41 + if (str) { + str = str.replace("共找到", "").replace("个小区", "").trim() + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count))) + println("当前处理城市" + obj.city) + } + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + } + + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "58.com"), new BasicDBObject("district_count", 1)) + .batchSize(100).each { obj -> + if (obj.district_count) { + number += obj.district_count + } + } + return number + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 28ac4a0ab..016d7be75 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,7 +29,7 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { def types = listAllType() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy new file mode 100644 index 000000000..be70b79e5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -0,0 +1,128 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取单线程方式 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiSingal { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(30).each {objTemp -> + final def obj = objTemp + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} From 7fe4a709f5d848225d9474fee6e5cc7a6902cd8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:16:14 +0800 Subject: [PATCH 059/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 5 ++ .../crawl/58/DistrictDetailCrawl.groovy | 46 +++++++++++++++++++ .../temp/shenjian/util/MessageUtils.groovy | 45 ++++++++++++++++++ .../fangjia/script/util/CommonUtils.groovy | 11 ++++- 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 0d5adfa99..871548bdf 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -487,5 +487,10 @@ 1.18.20 provided + + org.msgpack + msgpack + 0.6.12 + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy new file mode 100644 index 000000000..095456eed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 58小区详情爬取 + * + * @author shenjian + * @since 2021/5/26 + */ +class DistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { obj -> + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String)content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + println("已抓取" + obj.district_id) + } + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy new file mode 100644 index 000000000..93174d945 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSON +import org.msgpack.MessagePack +import org.msgpack.type.Value; + +/** + * msg解析工具类 + */ +public class MessageUtils { + + public static String unpack(byte[] content) { + MessagePack messagePack = new MessagePack(); + try { + Value value = messagePack.read(content); + return value.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + + public static byte[] pack(String value){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(value) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + public static byte[] pack(JSON json){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(json) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index 3d1d74c0c..d37680fac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -167,7 +167,16 @@ public class CommonUtils { } - + static void sendMessage(String queueName, byte[] bytes){ + ConnectionFactory factory = new ConnectionFactory(); + factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); + Connection conn = factory.newConnection(); + Channel channel = conn.createChannel(); + channel.queueDeclare(queueName, true, false, false, null); + channel.basicPublish("", queueName, null, bytes); + channel.close() + conn.close() + } /** From 019a18b28e30cc0a4bd6990b3c097238cd1bcc60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:17:52 +0800 Subject: [PATCH 060/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crawl/58/District58DetailCrawl.groovy | 67 +++++++++++++++++++ .../crawl/58/DistrictDetailCrawl.groovy | 46 ------------- 2 files changed, 67 insertions(+), 46 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..ca7a1b7bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +/** + * 58小区详情爬取 + * + * @author shenjian + * @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("District58DetailCrawl") + long count = 1; + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { obj -> + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String)content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + if (count % 20 == 0) { + stopWatch.stop() + log.info("抓取总量为{}, 耗时{}", count, stopWatch.getTotalTimeSeconds() / 60) + stopWatch.start() + } + count++ + log.info("已抓取 {}", obj.district_id) + } + } + } + } + if (stopWatch.isRunning()) { + stopWatch.stop() + log.info("抓取完毕。。。抓取总量为{}, 耗时{}", count++, stopWatch.getTotalTimeSeconds() / 60) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy deleted file mode 100644 index 095456eed..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy +++ /dev/null @@ -1,46 +0,0 @@ -package com.fangjia.script.temp.shenjian.crawl - -import com.alibaba.fastjson.JSON -import com.alibaba.fastjson.JSONObject -import com.fangjia.script.temp.shenjian.util.MessageUtils -import com.fangjia.script.util.ABuYunUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection - -/** - * 58小区详情爬取 - * - * @author shenjian - * @since 2021/5/26 - */ -class DistrictDetailCrawl { - - static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) - - static void main(String[] args) { - districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) - .batchSize(1000).limit(1000).each { obj -> - final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" - def res = ABuYunUtil.getReponseContent2(URL, "utf-8") - if (res && res.code == 200) { - String content = res.content - - JSON json = new JSONObject() - json.put("project", "district_detail_58") - json.put("url", URL) - - JSON result = new JSONObject() - result.put("content", (String)content) - json.put("result", result) - - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) - districtSeedConn.update(new BasicDBObject("_id", obj._id), - new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) - println("已抓取" + obj.district_id) - } - } - } -} \ No newline at end of file From bfbcdd07fa9c1cfa6115b4eb5f4e733166c9c81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:17:52 +0800 Subject: [PATCH 061/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 32 ++++++-- .../crawl/58/District58DetailCrawl.groovy | 73 +++++++++++++++++++ .../crawl/58/DistrictDetailCrawl.groovy | 46 ------------ .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 7 +- .../temp/shenjian/util/GaoDeUtils.groovy | 8 +- 5 files changed, 112 insertions(+), 54 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e95cbadff..10e63221e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -2,6 +2,11 @@ package com.fangjia.script.temp.shenjian import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject +import com.baidubce.http.ApiExplorerClient +import com.baidubce.http.AppSigner +import com.baidubce.http.HttpMethodName +import com.baidubce.model.ApiExplorerRequest +import com.baidubce.model.ApiExplorerResponse import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.SeaweedUtils import org.elasticsearch.geometry.utils.Geohash @@ -12,11 +17,28 @@ import org.elasticsearch.geometry.utils.Geohash class Temp { static void main(String[] args) { - String str = SeaweedUtils.encodeByGeoHash(106.569161, 29.498502) - println(str) - str = Geohash.stringEncode(106.569161, 29.498502, 10) - println(str) - // wm5zc4131rmf + String path = "https://goojia.api.bdymkt.com/estimate"; + ApiExplorerRequest request = new ApiExplorerRequest(HttpMethodName.GET, path); + request.setCredentials("fccf809a0f7748449154af926897aecb", "af1f0088a05c493ab20133ba470f87ef "); + + // 设置header参数 + request.addHeaderParameter("Content-Type", "application/json; charset=utf-8"); + + // 设置query参数 + request.addQueryParameter("address", "上海宝山呼玛三村"); + request.addQueryParameter("city", "上海"); + request.addQueryParameter("size", "100"); + + + ApiExplorerClient client = new ApiExplorerClient(new AppSigner()); + + try { + ApiExplorerResponse response = client.sendRequest(request); + // 返回结果格式为Json字符串 + System.out.println(response.getResult()); + } catch (Exception e) { + e.printStackTrace(); + } } // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..d02c980c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,73 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 58小区详情爬取 + * + * @author shenjian* @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(100000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy deleted file mode 100644 index 095456eed..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy +++ /dev/null @@ -1,46 +0,0 @@ -package com.fangjia.script.temp.shenjian.crawl - -import com.alibaba.fastjson.JSON -import com.alibaba.fastjson.JSONObject -import com.fangjia.script.temp.shenjian.util.MessageUtils -import com.fangjia.script.util.ABuYunUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection - -/** - * 58小区详情爬取 - * - * @author shenjian - * @since 2021/5/26 - */ -class DistrictDetailCrawl { - - static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) - - static void main(String[] args) { - districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) - .batchSize(1000).limit(1000).each { obj -> - final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" - def res = ABuYunUtil.getReponseContent2(URL, "utf-8") - if (res && res.code == 200) { - String content = res.content - - JSON json = new JSONObject() - json.put("project", "district_detail_58") - json.put("url", URL) - - JSON result = new JSONObject() - result.put("content", (String)content) - json.put("result", result) - - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) - districtSeedConn.update(new BasicDBObject("_id", obj._id), - new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) - println("已抓取" + obj.district_id) - } - } - } -} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 016d7be75..4d7a51343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,11 +29,10 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(10) + static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { def types = listAllType() - //def types = "0601" DBObject query = new BasicDBObject() query.put("status", 1) conn136.find(query).batchSize(5000).each {objTemp -> @@ -46,6 +45,10 @@ class SeaweedAroundPoi { def lat = obj.lat List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") if (CollectionUtils.isNotEmpty(pois)) { + if (pois.get(0).id == "10044") { + executorService.shutdown() + throw new Error("当日调用已超限") + } String jsonStr = JSONObject.toJSONString(pois) HttpUtils.postForJson(URL, jsonStr) conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index e384c35fb..3f234cc53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -8,6 +8,8 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import java.util.concurrent.ExecutorService + /** * 高德工具类 * @@ -138,7 +140,11 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10044") { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 - System.exit(1) + //System.exit(1) + AmapPoi amapPoi = new AmapPoi() + amapPoi.id = "10044" + result.add(amapPoi) + break; } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) From 6c19fafa521e307811403c8d00026199fd46f920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 28 May 2021 09:08:31 +0800 Subject: [PATCH 062/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AE=89=E5=B1=85?= =?UTF-8?q?=E5=AE=A2=E5=B0=8F=E5=8C=BA=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E6=8A=93=E5=8F=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../anjuke/AnjukeDistrictDetailCrawl.groovy | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy new file mode 100644 index 000000000..c9caccb1a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.anjuke + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 安居客小区详情抓取 + * + * @author shenjian + * @since 2021/5/28 + */ +@Slf4j +class AnjukeDistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "anjuke.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".anjuke.com/community/view/" + obj.district_id + "#/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("请拖动滑块进行验证") == -1 && content.indexOf("访问验证-安居客") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_anjuke") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } else if (res && res.code == 508) { + log.info("508 {}", obj.district_id) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +} From 62a844316074db7f08e4ea039d4b165e48108bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 1 Jun 2021 16:01:48 +0800 Subject: [PATCH 063/118] =?UTF-8?q?=E7=BB=A9=E6=95=88=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B9=E7=9B=AE=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/dingding/TaskExport.groovy | 5 +---- .../script/temp/shenjian/util/GaoDeUtils.groovy | 10 +++++----- .../java/com/fangjia/script/statistics/ApiUtil.java | 4 +++- .../main/java/com/fangjia/script/statistics/Task.java | 11 +++++++++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index a86e1a959..1cc8d65b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -49,13 +49,10 @@ public class TaskExport { Thread.sleep(60 * 1000) taskDetail = ApiUtil.getTaskDetail(task.getId()) } - if ("穆良书" == task.getResponse().getName() && task.getId() == "e3f81b0381b44696b06fb6242a44902f") { - print("穆良书") - } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { TaskPersonDTO dto = new TaskPersonDTO() dto.name = task.response.name - dto.project = "" + dto.project = task.project.name dto.taskName = task.name dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten) dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 3f234cc53..e16b6336a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -140,11 +140,11 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10044") { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 - //System.exit(1) - AmapPoi amapPoi = new AmapPoi() - amapPoi.id = "10044" - result.add(amapPoi) - break; + System.exit(1) +// AmapPoi amapPoi = new AmapPoi() +// amapPoi.id = "10044" +// result.add(amapPoi) +// break; } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 977bc2888..7c87580de 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -340,7 +340,9 @@ public class ApiUtil { JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data"))); detail = obj.toJavaObject(TaskDetail.class); } else if (code == 90001){ - throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:getTaskDetail"); + Thread.sleep(1000 * 60); + return getTaskDetail(taskId); + //throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:getTaskDetail"); } else if (code == 90009) { throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:getTaskDetail"); } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java index 9a0c728c4..78fd1a825 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java @@ -32,6 +32,8 @@ public class Task { private FlowDTO flow; @JSONField(name = "createDate") private String createDate; + @JSONField(name = "project") + private ProjectDTO project; @NoArgsConstructor @Data @@ -61,4 +63,13 @@ public class Task { @JSONField(name = "userid") private String userid; } + + @NoArgsConstructor + @Data + public static class ProjectDTO { + @JSONField(name = "id") + private String id; + @JSONField(name = "proname") + private String name; + } } From ea3ae0cf612a1c1a52973f332c74114d82142790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 13:44:09 +0800 Subject: [PATCH 064/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E5=9F=8E=E5=B8=82=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/58/City58.groovy | 6 +- .../temp/shenjian/crawl/ke/CityKe.groovy | 57 +++++++++++++++++++ .../temp/shenjian/util/CityUtils.groovy | 19 +++++++ 3 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy index 8b9abf395..5222957f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.temp.shenjian.crawl import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.CityUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -252,16 +253,13 @@ class City58 { " }"; static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) - static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static Set citySet = CityUtils.listCitySet() static void main(String[] args) { JSONObject jsonObject = JSONObject.parseObject(cityList) Set provinceSet = jsonObject.keySet() List ret = new ArrayList<>() - List list = cityMapConn.distinct("fangjia_city") - Set citySet = new HashSet<>(list); - for (String province: provinceSet) { JSONObject provinceInfo = jsonObject[province] for (String city : provinceInfo.keySet()) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy new file mode 100644 index 000000000..e8ae7fc0c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + final def URL = "https://www.ke.com/city/" + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + Elements elements = JQueryUtils.select(".city_list_ul .city_list_li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText(".city_province .city_list_tit", element) + Elements cityElements = JQueryUtils.select(".CLICKDATA a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".ke.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "ke.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("贝壳找房城市数据写入成功") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy new file mode 100644 index 000000000..2949d3803 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy @@ -0,0 +1,19 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +/** + * @author shenjian + * @since 2021/6/2 + */ +class CityUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static listCitySet() { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + return citySet + } +} From 3395ae6248ebe35e8d04555949751ddde4163def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 14:08:19 +0800 Subject: [PATCH 065/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E5=90=84=E5=9F=8E=E5=B8=82=E6=88=90=E4=BA=A4?= =?UTF-8?q?=E6=80=BB=E9=87=8F=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/ke/CountTradeKe.groovy | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy new file mode 100644 index 000000000..064598c6b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 贝壳各城市成交总量抓取 + * + * @author shenjian + * @since 2021/6/2 + */ +class CountTradeKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + crawlTradeCount() + //println(count()) + } + + static crawlTradeCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { objTemp -> + final def obj = objTemp + executorService.execute(new Runnable() { + @Override + void run() { + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".ke.com/chengjiao/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + boolean existTrade = JQueryUtils.selectText(".resultDes .total", doc).contains("成交房源") + if (existTrade) { + String str = JQueryUtils.selectText(".resultDes .total span", doc) + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", count))) + } else { + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", 0))) + } + println("当前处理城市" + obj.city) + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + }) + } + + executorService.shutdown() + } + + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1)) + .batchSize(100).each { obj -> + if (obj.trade_count) { + number += obj.trade_count + } + } + return number + } +} From 62a419ea669e6bbe34c88fe2510c3244c40aea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 14:36:29 +0800 Subject: [PATCH 066/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E6=8A=93=E5=8F=96=E5=BC=82=E5=B8=B8=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/ke/CountTradeKe.groovy | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy index 064598c6b..d8b2c466b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -20,11 +20,14 @@ import java.util.concurrent.Executors class CountTradeKe { static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - crawlTradeCount() + //crawlTradeCount() //println(count()) + //1896919 1760177 + findUnNormalCount() } static crawlTradeCount() { @@ -62,6 +65,28 @@ class CountTradeKe { executorService.shutdown() } + /** + * 找出异常抓取的城市 + */ + static findUnNormalCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1)) + .batchSize(100).each { obj -> + Long realTradeCount = obj.trade_count + if (realTradeCount > 0) { + Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房")) + if (realTradeCount > crawledCount) { + println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount)) + } + } + } + } + + /** + * 统计实际抓取量 + * + * @return + */ static count() { int number = 0; cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1)) From 37d0ad878c6166d77300c92e4b5d9cf908f99c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 16:53:12 +0800 Subject: [PATCH 067/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=88=BF=E5=A4=A9?= =?UTF-8?q?=E4=B8=8B=E5=9F=8E=E5=B8=82=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/fang/CityFang.groovy | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy new file mode 100644 index 000000000..88a3b6f38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityFang { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + // 房天下较为特殊,直接从url取乱码,需要改页面所有非UTF-8编码为UTF-8 + Document doc = Jsoup.parse(new File("F:\\数据\\爬虫\\fang_city.html"), "UTF-8"); + Elements elements = JQueryUtils.select(".outCont li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText("strong", element) + Elements cityElements = JQueryUtils.select("a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if ("直辖市".equals(province)) { + province = city + } + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".esf.fang.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "fang.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("房天下城市数据写入成功") + } +} From 995e92f4382f69edc811f0959497bc949ba17d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 4 Jun 2021 10:53:46 +0800 Subject: [PATCH 068/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=87=8F=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/seaweed/CountSeaweed.groovy | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy new file mode 100644 index 000000000..ff7f7d322 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * @author shenjian + * @since 2021/6/4 + */ +class CountSeaweed { + static void main(String[] args) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("cat", "district"); + q.put("visible", 0); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + println(city + " " + DbUtils.seaweed.count(q)) + } + + } +} From 58f41e5154647764c94ef378d105bbeb317b5caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 7 Jun 2021 10:14:21 +0800 Subject: [PATCH 069/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=87=8F=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy | 4 ---- .../com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 4 ---- 2 files changed, 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 4d7a51343..a0227abe4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -45,10 +45,6 @@ class SeaweedAroundPoi { def lat = obj.lat List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") if (CollectionUtils.isNotEmpty(pois)) { - if (pois.get(0).id == "10044") { - executorService.shutdown() - throw new Error("当日调用已超限") - } String jsonStr = JSONObject.toJSONString(pois) HttpUtils.postForJson(URL, jsonStr) conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index e16b6336a..494e0cb22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -141,10 +141,6 @@ class GaoDeUtils { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 System.exit(1) -// AmapPoi amapPoi = new AmapPoi() -// amapPoi.id = "10044" -// result.add(amapPoi) -// break; } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) From 2da2fb9e4216b4c2a5b8a2abd649cef80d0b1492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 7 Jun 2021 13:57:12 +0800 Subject: [PATCH 070/118] =?UTF-8?q?=E4=BC=98=E5=85=88=E5=A4=84=E7=90=86?= =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86=E8=A1=8C=E6=8B=9B=E6=A0=87?= =?UTF-8?q?=E5=9F=8E=E5=B8=82POI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index a0227abe4..addf2d9cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -33,7 +33,36 @@ class SeaweedAroundPoi { static void main(String[] args) { def types = listAllType() - DBObject query = new BasicDBObject() + dealSpecialSeaweedAround(types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { query.put("status", 1) conn136.find(query).batchSize(5000).each {objTemp -> final def obj = objTemp @@ -55,7 +84,6 @@ class SeaweedAroundPoi { } }) } - executorService.shutdown() } static getNewSeaweed() { From e4ac06de8c236df297935752f1d2a5e9fe6cb924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 11 Jun 2021 15:27:00 +0800 Subject: [PATCH 071/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 58 ++++++++++++++++++- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 3 +- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index 8627efc19..34df24342 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.time.DateFormatUtils /** * 抓取总量监控 @@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch { static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] static void main(String[] args) { - runBatch(new SeawaterMonitor(), args) + //runBatch(new SeawaterMonitor(), args) + //monitorDayCrawlReasonable() + def results = [] + results << ["日期", "网站", "出售", "出租"] + for (int i = 1; i < 90; i++) { + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime() + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + countDayCrawlBySite(startDate, endDate, results) + } + ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1") } /** @@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch { return "" } + /** + * 统计近90天更新量超过40万的天数 + * @param args + */ + static def monitorDayCrawlReasonable() { + def map = [:] + for (int i = 1; i < 90; i++) { + def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i) + def startDate = date.getTime(); + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + def total = countDayCrawl(startDate, endDate) + if (total >= expectNumber) { + map.put(DateUtil.formatToYYYYMMDD(date), total) + } + } + println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" ) + for (def key : map.keySet()) { + println(key + " : " + map.get(key)) + } + } + + /** + * 统计每个网站时间段范围内的更新量 + */ + static def countDayCrawlBySite(def startDate, def endDate, def results) { + def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"] + def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat" + def map = [:] + ElasticUtils.executeSqlQuery(sql,{data-> + map.put(data.root + "|" + data.cat, data.total) + }); + for (def site : sites) { + results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")] + } + return results + }; + + static def countDayCrawl(def startDate, def endDate) { + def total = 0 + def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'" + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + return total + }; + @Override void process(Object args) { def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index addf2d9cf..7a10c9035 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -33,7 +33,8 @@ class SeaweedAroundPoi { static void main(String[] args) { def types = listAllType() - dealSpecialSeaweedAround(types) + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) executorService.shutdown() } From f51d9ed812d3bebfe95d0b7929e530219f243c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 17:32:03 +0800 Subject: [PATCH 072/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=AB=98=E5=BE=B7key?= =?UTF-8?q?=E5=A4=9A=E7=BA=BF=E7=A8=8B=E4=BD=BF=E7=94=A8=E5=AE=8C=E5=81=9C?= =?UTF-8?q?=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 494e0cb22..5adde74e3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -8,7 +8,7 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import java.util.concurrent.ExecutorService +import java.util.concurrent.atomic.AtomicInteger /** * 高德工具类 @@ -19,7 +19,8 @@ import java.util.concurrent.ExecutorService class GaoDeUtils { // 新获取key - def static GAO_DE_AK = "4e9592208336433ee1b7aa4d1d434418" + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) def static codeNameMap = new HashMap(){{ put("150500", "地铁"); @@ -59,7 +60,7 @@ class GaoDeUtils { */ static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] - params.key = GAO_DE_AK + params.key = GAO_DE_AK[keyUsed] if (types) { params.types = types @@ -108,7 +109,7 @@ class GaoDeUtils { */ static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] - params.key = GAO_DE_AK + params.key = GAO_DE_AK[keyUsed.intValue()] if (types) { params.types = types @@ -140,7 +141,10 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10044") { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 - System.exit(1) + if (keyUsed.intValue() >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed.incrementAndGet() } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) From b7ae62d5821c90ef563d4e88e114243292cd6b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 17:59:35 +0800 Subject: [PATCH 073/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9POI?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 7a10c9035..476a42a3a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -1,20 +1,16 @@ package com.fangjia.script.temp.shenjian.poi - import com.alibaba.fastjson.JSONObject -import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.MongoUtils -import com.fangjia.script.wenyan.auto.KeywordSearchUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.commons.collections.CollectionUtils -import org.bson.types.ObjectId import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -154,10 +150,24 @@ class SeaweedAroundPoi { static def listAllType() { StringBuilder ret = new StringBuilder() - for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + for (def val : KEYWORDS_MAP.values()) { ret.append(val).append("|") } String result = ret.toString() return result.substring(0, result.length() - 1) } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; } From 6d6c7cab29c33628c611768f5c3ffc9bfa96a485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 18:08:24 +0800 Subject: [PATCH 074/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9POI?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 185 ++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 181 +++++++++++++++ .../temp/shenjian/util/HttpUtils.groovy | 29 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 208 ++++++++++++++++++ .../script/statistics/poi/AmapPoi.java | 71 ++++++ 6 files changed, 680 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..75ca2b1ff 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -482,7 +482,12 @@ j2v8_win32_x86_64 4.6.0 - + + org.projectlombok + lombok + 1.18.20 + provided + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..00ee277ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..5adde74e3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,181 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + // 多线程情况下还是exit有效果,抛异常不会停止程序 + if (keyUsed.intValue() >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed.incrementAndGet() + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..9fa4739f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,208 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2000 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From 7042809ecd9bf1fa3af87232fa71c16f124fd914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 18:08:24 +0800 Subject: [PATCH 075/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E5=A4=9Akey=E5=81=9C=E6=AD=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 185 ++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 186 ++++++++++++++++ .../temp/shenjian/util/HttpUtils.groovy | 29 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 208 ++++++++++++++++++ .../script/statistics/poi/AmapPoi.java | 71 ++++++ 6 files changed, 685 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..75ca2b1ff 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -482,7 +482,12 @@ j2v8_win32_x86_64 4.6.0 - + + org.projectlombok + lombok + 1.18.20 + provided + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..00ee277ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..74505e6b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,186 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..9fa4739f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,208 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2000 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From 975dc9094752124015d68757b16e2474654471e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 22 Jun 2021 13:45:54 +0800 Subject: [PATCH 076/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=8C=96=E6=89=B9=E9=87=8F=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/address/StandardTest.groovy | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy new file mode 100644 index 000000000..3b3cca4f5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy @@ -0,0 +1,60 @@ +package com.fangjia.script.temp.shenjian.address + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 地址标准化测试 + * + * @author shenjian + * @since 2021/6/22 + */ +class StandardTest { + + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong(0) + List> data = ExcelUtils.read("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据.xlsx") + CountDownLatch latch = new CountDownLatch(data.size()) + for (List rowT : data) { + final row = rowT; + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = row[0] + def city = row[2] + HttpRequest request = new HttpRequest() + + String result = request.get("http://192.168.20.86:8082/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") + JSONObject object = JSONObject.parse(result); + if (object.get("status") == 0) { + row[7] = (((JSONObject) object.get("result")).get("formatAddress")) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + count.incrementAndGet() + if (count.intValue() % 1000 == 0) { + Thread.sleep(5000) + println(count) + } + } + } + }) + } + latch.await() + def headers = ["原始地址", "XH", "城市", "区域","小区名", "标注", "备注", "标准化地址"] + data.add(0, headers) + ExcelUtils.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx", data, "测试数据") + executorService.shutdown() + } +} From f4bd346bb43a0b733fb8519df3a6f272b75afabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 22 Jun 2021 14:21:37 +0800 Subject: [PATCH 077/118] =?UTF-8?q?=E4=BB=8E=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E5=BE=85=E6=A0=87=E5=87=86=E5=8C=96=E8=BE=93?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/ExportStandardResult.groovy | 63 +++++++++++++++++++ .../temp/shenjian/address/StandardTest.groovy | 25 ++++---- 2 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy new file mode 100644 index 000000000..bc2365983 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy @@ -0,0 +1,63 @@ +package com.fangjia.script.temp.shenjian.address + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 导出标准化输出结果 + * + * @author shenjian + * @since 2021/6/22 + */ +class ExportStandardResult { + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) + + static void main(String[] args) { + List> data = new ArrayList<>() + def map = new ConcurrentHashMap() + CountDownLatch latch = new CountDownLatch(conn136.count().intValue()) + conn136.find(new BasicDBObject(), new BasicDBObject("address", 1).append("city", 1) + .append("format_address", 1).append("order", 1)).batchSize(10000).each { + objT -> + final obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def ret = [] + ret[0] = obj.city + ret[1] = obj.address + ret[2] = obj.format_address + map.put(obj.order, ret) + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + for (int i = 0; i < map.size(); i++) { + data.add(map.get(i)) + } + def headArr = ["城市", "地址", "标准化输出"] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + EasyExcel.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx").head(head).sheet("测试数据").doWrite(data); + + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy index 3b3cca4f5..b42c0b70d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy @@ -3,6 +3,9 @@ package com.fangjia.script.temp.shenjian.address import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -18,25 +21,28 @@ import java.util.concurrent.atomic.AtomicLong class StandardTest { static ExecutorService executorService = Executors.newFixedThreadPool(10) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) static void main(String[] args) { AtomicLong count = new AtomicLong(0) - List> data = ExcelUtils.read("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据.xlsx") - CountDownLatch latch = new CountDownLatch(data.size()) - for (List rowT : data) { - final row = rowT; + CountDownLatch latch = new CountDownLatch(conn136.count(new BasicDBObject('status', 1)).intValue()) + conn136.find(new BasicDBObject('status', 1)).batchSize(3000).each {objT -> + final def obj = objT executorService.submit(new Runnable() { @Override void run() { try { - def address = row[0] - def city = row[2] + def address = obj.address + def city = obj.city == null ? "" : obj.city HttpRequest request = new HttpRequest() String result = request.get("http://192.168.20.86:8082/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") JSONObject object = JSONObject.parse(result); if (object.get("status") == 0) { - row[7] = (((JSONObject) object.get("result")).get("formatAddress")) + def format_address = (((JSONObject) object.get("result")).get("formatAddress")) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("format_address", format_address).append("flag", "已标准化输出"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "标准化输出异常"))) } } catch(Exception ex) { ex.printStackTrace() @@ -50,11 +56,8 @@ class StandardTest { } } }) - } + }; latch.await() - def headers = ["原始地址", "XH", "城市", "区域","小区名", "标注", "备注", "标准化地址"] - data.add(0, headers) - ExcelUtils.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx", data, "测试数据") executorService.shutdown() } } From 714751cc9adf45156d5ddac155680bd11189c087 Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Tue, 22 Jun 2021 18:39:01 +0800 Subject: [PATCH 078/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=89=80=E8=AF=B7=E6=B1=82=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E4=B8=8D=E5=8F=AF=E7=94=A8=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 74505e6b5..002e6aa14 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -154,6 +154,9 @@ class GaoDeUtils { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) } else { break } From a8270b4f6825a110b33309fa3eb3a2d3f524b157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 28 Jun 2021 16:00:03 +0800 Subject: [PATCH 079/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A0=87=E5=87=86?= =?UTF-8?q?=E5=8C=96=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/ExportStandardResult.groovy | 31 +++++++++--- .../temp/shenjian/address/StandardTest.groovy | 50 ++++++++++++++++--- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy index bc2365983..7b400357a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy @@ -4,6 +4,7 @@ import com.alibaba.excel.EasyExcel import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -23,9 +24,10 @@ class ExportStandardResult { static void main(String[] args) { List> data = new ArrayList<>() def map = new ConcurrentHashMap() - CountDownLatch latch = new CountDownLatch(conn136.count().intValue()) - conn136.find(new BasicDBObject(), new BasicDBObject("address", 1).append("city", 1) - .append("format_address", 1).append("order", 1)).batchSize(10000).each { + // 查询条件 + DBObject query = new BasicDBObject('status', 0).append('flag', '浙商疑问地址复测') + CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue()) + conn136.find(query).batchSize(10000).each { objT -> final obj = objT executorService.submit(new Runnable() { @@ -33,9 +35,22 @@ class ExportStandardResult { void run() { try { def ret = [] - ret[0] = obj.city - ret[1] = obj.address - ret[2] = obj.format_address + ret << obj.format_province + ret << obj.format_city + ret << obj.format_region + ret << obj.format_dev + ret << obj.format_township + ret << obj.format_community + ret << obj.format_group + ret << obj.format_street + ret << obj.format_streetNumber + ret << obj.format_name + ret << obj.format_buildingNumber + ret << obj.format_unitNumber + ret << obj.format_roomNumber + ret << obj.format_floor + ret << obj.format_address + map.put(obj.order, ret) } catch (Exception ex) { ex.printStackTrace() @@ -49,14 +64,14 @@ class ExportStandardResult { for (int i = 0; i < map.size(); i++) { data.add(map.get(i)) } - def headArr = ["城市", "地址", "标准化输出"] + def headArr = ["省","市","区","开发区","街道乡镇","居委会/社区", "组","路","路号","名称","楼栋", "单元","室号","楼层","优化后标准化地址"] List head = new ArrayList<>() for (String name: headArr) { List temp = new ArrayList<>() temp.add(name) head.add(temp) } - EasyExcel.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx").head(head).sheet("测试数据").doWrite(data); + EasyExcel.write("F:\\工作\\基础建设\\标准化\\浙商疑问地址复测.xlsx").head(head).sheet("测试数据").doWrite(data); executorService.shutdown() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy index b42c0b70d..8611995d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -22,11 +23,13 @@ class StandardTest { static ExecutorService executorService = Executors.newFixedThreadPool(10) static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) - + static final String IP = "192.168.0.19:39080" static void main(String[] args) { AtomicLong count = new AtomicLong(0) - CountDownLatch latch = new CountDownLatch(conn136.count(new BasicDBObject('status', 1)).intValue()) - conn136.find(new BasicDBObject('status', 1)).batchSize(3000).each {objT -> + // 查询条件 + DBObject query = new BasicDBObject('status', 1).append('flag', '浙商疑问地址复测') + CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue()) + conn136.find(query).batchSize(3000).each {objT -> final def obj = objT executorService.submit(new Runnable() { @Override @@ -36,11 +39,44 @@ class StandardTest { def city = obj.city == null ? "" : obj.city HttpRequest request = new HttpRequest() - String result = request.get("http://192.168.20.86:8082/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") + String result = request.get("http://" + IP + "/FangJiaAddressService/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") JSONObject object = JSONObject.parse(result); if (object.get("status") == 0) { - def format_address = (((JSONObject) object.get("result")).get("formatAddress")) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("format_address", format_address).append("flag", "已标准化输出"))) + JSONObject jsonResult = (JSONObject) object.get("result"); + def format_province = jsonResult.get("province") + def format_city = jsonResult.get("city") + def format_region = jsonResult.get("district") + def format_dev = jsonResult.get("dev") + def format_township = jsonResult.get("township") + def format_community = jsonResult.get("community") + def format_group = jsonResult.get("group") + def format_street = jsonResult.get("street") + def format_subStreet = jsonResult.get("subStreet") + def format_streetNumber = jsonResult.get("streetNumber") + def format_name = jsonResult.get("name") + def format_buildingNumber = jsonResult.get("buildingNumber") + def format_unitNumber = jsonResult.get("unitNumber") + def format_floor = jsonResult.get("floor") + def format_roomNumber = jsonResult.get("roomNumber") + def format_address = jsonResult.get("formatAddress") + + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0) + .append("format_province", format_province) + .append("format_city", format_city) + .append("format_region", format_region) + .append("format_dev", format_dev) + .append("format_township", format_township) + .append("format_community", format_community) + .append("format_group", format_group) + .append("format_street", format_street) + .append("format_subStreet", format_subStreet) + .append("format_streetNumber", format_streetNumber) + .append("format_name", format_name) + .append("format_buildingNumber", format_buildingNumber) + .append("format_unitNumber", format_unitNumber) + .append("format_floor", format_floor) + .append("format_roomNumber", format_roomNumber) + .append("format_address", format_address))) } else { conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "标准化输出异常"))) } @@ -57,7 +93,7 @@ class StandardTest { } }) }; - latch.await() + latch.await(); executorService.shutdown() } } From e788d211f1e317a47cdd918ae3bfacaacee13c4a Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 1 Jul 2021 18:08:11 +0800 Subject: [PATCH 080/118] =?UTF-8?q?=E7=BB=A9=E6=95=88=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy | 2 +- .../fangjia/script/temp/shenjian/dingding/ProjectExport.groovy | 2 +- .../fangjia/script/temp/shenjian/dingding/ScoreExport.groovy | 2 +- .../com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 00e7644b8..9603351a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,7 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-04-01", "yyyy-MM-dd") ; + Date now = DateUtils.parseFromTightByPattern("2021-06-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy index efd9a2e3b..b2535679e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy @@ -93,7 +93,7 @@ class ProjectExport { mergeData.add(mergeRowCel) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy index 511c72846..6a10fff7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy @@ -140,7 +140,7 @@ public class ScoreExport { allTotal[2] = "人员绩效总计" sortedResult << allTotal - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 1cc8d65b5..d7958a211 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -117,7 +117,7 @@ public class TaskExport { mergeData.add(data) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData); } } From 37c2ee5c89681e975cf034e790749e33b9ba110d Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 1 Jul 2021 18:19:35 +0800 Subject: [PATCH 081/118] =?UTF-8?q?=E6=96=B0=E5=A2=9Eexcel=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=E5=B7=A5=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/util/ExcelUtils.groovy | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy new file mode 100644 index 000000000..229499b80 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.excel.EasyExcelFactory +import com.alibaba.excel.ExcelReader +import com.alibaba.excel.context.AnalysisContext +import com.alibaba.excel.event.AnalysisEventListener + +/** + * excel工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class ExcelUtils { + + static List> read(String filePath) { + SimpleExcelListener listener = new SimpleExcelListener(filePath); + ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); + excelReader.readAll() + List> dataList = listener.getDataList(); + excelReader.finish() + return dataList; + } + + class SimpleExcelListener extends AnalysisEventListener { + + private List> dataList = new ArrayList<>() + + @Override + void invoke(Object o, AnalysisContext analysisContext) { + Map map = o; + dataList.add(new ArrayList(map.values())) + } + + @Override + void doAfterAllAnalysed(AnalysisContext analysisContext) { + + } + + List> getDataList() { + return dataList + } + + void setDataList(List> dataList) { + this.dataList = dataList + } + } +} From 6b58f0a85629eeed95a9d400d9c4cfe18d7c5690 Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 1 Jul 2021 18:19:35 +0800 Subject: [PATCH 082/118] =?UTF-8?q?=E6=96=B0=E5=A2=9EPOI=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/dingding/ScoreExport.groovy | 3 - .../temp/shenjian/poi/StatisticsPoi.groovy | 59 +++++++++++++++++++ .../temp/shenjian/util/ExcelUtils.groovy | 48 +++++++++++++++ .../temp/shenjian/util/MyStringUtils.groovy | 40 +++++++++++++ 4 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy index 6a10fff7c..2218e2ec6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy @@ -69,9 +69,6 @@ public class ScoreExport { } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { boolean find = false - if (taskDetail.getResponse().getName() == '穆良书' && taskDetail.getName() == "模型级联算法方案设计") { - println('穆良书') - } for (int i = 4; i < sortedResult[0].size(); i++) { // 找到该项目下该负责人,则加工时 if (sortedResult[0][i] == taskDetail.getResponse().getName()) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy new file mode 100644 index 000000000..cc0cb39dd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.MyStringUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +/** + * @author shenjian* @since 2021/7/7 + */ +class StatisticsPoi { + static final cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京", + "青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原", + "台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳", + "哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江", + "岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳", + "柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照", + "拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + + + static final types = ['地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆'] + + static void main(String[] args) { + def result = [] + + result << ['城市', '数据量', '地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆'] + for (def city in cities) { + def ret = [] + def total + String sql = "select count(*) total from amap_poi where cityname = '" + city + "' and status <> 1 " + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.total + }) + ret << city + ret << total + for (def type in types) { + def typeCode = SeaweedAroundPoi.KEYWORDS_MAP.get(type) + def typeCodeArr = typeCode.split("\\|") + String val = "" + String sqlType = "" + if (typeCodeArr[0].size() <= 4) { + val = MyStringUtils.joinLikeSql(typeCodeArr) + sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and (" + val + ") and status <> 1 " + } else { + val = MyStringUtils.joinInSql(typeCodeArr) + sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and typecode in (" + val + ") and status <> 1 " + } + + ElasticUtils.executeSqlQuery(sqlType,{ data-> + total = data.total + }) + ret << total + } + result << ret + println(city) + } + + ExcelUtils.write("F:\\工作\\房产估值模型及数据招标\\交付工作\\POI信息抓取情况总结.xlsx", result, "POI统计") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy new file mode 100644 index 000000000..229499b80 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.excel.EasyExcelFactory +import com.alibaba.excel.ExcelReader +import com.alibaba.excel.context.AnalysisContext +import com.alibaba.excel.event.AnalysisEventListener + +/** + * excel工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class ExcelUtils { + + static List> read(String filePath) { + SimpleExcelListener listener = new SimpleExcelListener(filePath); + ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); + excelReader.readAll() + List> dataList = listener.getDataList(); + excelReader.finish() + return dataList; + } + + class SimpleExcelListener extends AnalysisEventListener { + + private List> dataList = new ArrayList<>() + + @Override + void invoke(Object o, AnalysisContext analysisContext) { + Map map = o; + dataList.add(new ArrayList(map.values())) + } + + @Override + void doAfterAllAnalysed(AnalysisContext analysisContext) { + + } + + List> getDataList() { + return dataList + } + + void setDataList(List> dataList) { + this.dataList = dataList + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy new file mode 100644 index 000000000..36eb6c935 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.temp.shenjian.util + +/** + * @author shenjian* @since 2021/7/7 + */ +public class MyStringUtils { + + public static joinSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append("'").append(str[i]).append("'") + if (i != str.size() - 1) { + sb.append(",") + } + } + return sb.toString() + } + + public static joinInSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append("'").append(str[i]).append("'") + if (i != str.size() - 1) { + sb.append(",") + } + } + return sb.toString() + } + + public static joinLikeSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append(" typecode like '").append(str[i]).append("%' ") + if (i != str.size() - 1) { + sb.append(" or ") + } + } + return sb.toString() + } +} From 5574322e649f8d30250328d00b1897c532cde34d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 7 Jul 2021 14:05:46 +0800 Subject: [PATCH 083/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dexcel=E5=90=88?= =?UTF-8?q?=E5=B9=B6bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/util/ExcelUtils.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 203508a51..cc026d9fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -346,11 +346,13 @@ public class ExcelUtils{ mergedCellStyle.alignment = HorizontalAlignment.CENTER; for (List rowCell : data) { - CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3)) - sheet.addMergedRegion(cra); - //设置合并后style - def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2)); - cell.setCellStyle(mergedCellStyle) + if (rowCell.get(0) != rowCell.get(1)) { + CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3)) + sheet.addMergedRegion(cra); + //设置合并后style + def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2)); + cell.setCellStyle(mergedCellStyle) + } } } From 93a2ba322ce17d34bd6aaf6beb9157d3c9ba6070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 7 Jul 2021 16:22:09 +0800 Subject: [PATCH 084/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=91=A8=E8=BE=B9=E5=B1=85=E5=A7=94=E4=BC=9A=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../poi/SeaweedAroundCommittee.groovy | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy new file mode 100644 index 000000000..d8edf839d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边居委会获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundCommittee { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + // 乡镇以下级政府及事业单位(居委会) + def types = "130106" + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + static dealPoi(DBObject query, def types) { + query.put("status_committee", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 0).append("flag_committee", "已获取居委会"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 2).append("flag_committee", "周边居委会为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} From da6476465147eac5e24fffa6af2de02f8d86c78d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 11:41:50 +0800 Subject: [PATCH 085/118] =?UTF-8?q?=E5=9B=BD=E5=AE=B6=E6=A0=87=E5=87=86?= =?UTF-8?q?=E5=B1=85=E5=A7=94=E4=BC=9A=E7=BB=8F=E7=BA=AC=E5=BA=A6=E8=8E=B7?= =?UTF-8?q?=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/CommitteeLatLng.groovy | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy new file mode 100644 index 000000000..cadb279a7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -0,0 +1,50 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 国家标准居委会经纬度获取 + * + * @author shenjian + * @since 2021/7/22 + */ +class CommitteeLatLng { + + static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + villagesConn.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city_name + def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; + def location = GaoDeUtils.parserByGaode(city, address); + if (location && location.lng) { + def lng = location.lng + def lat = location.lat + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("lng", lng).append("lat", lat).append("flag_location", "已获取经纬度"))) + } else { + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} From 997e10c1d87de8990d57934e5a7329c05deb99f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 11:48:04 +0800 Subject: [PATCH 086/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 002e6aa14..9a3bc83d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -176,7 +176,7 @@ class GaoDeUtils { */ static def parserByGaode(def city, def address) { JSONObject json = GaoDeApiUtils.geo(city, address) - if (json && json.status == "1") { + if (json && json.info && json.info == "OK") { def ret = json.geocodes if (ret) { def str = ret.get(0).location @@ -184,6 +184,14 @@ class GaoDeUtils { def arrs = str.split(",") return ["lng": arrs[0], "lat": arrs[1]] } + } else if (json && json.infocode && json.infocode == "10044") { + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) } } } From 469f9a22f9fd2ab552ebf77d023db03dd186573c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 14:36:35 +0800 Subject: [PATCH 087/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 9a3bc83d4..4d7cee9f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.temp.shenjian.util import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil @@ -175,7 +176,17 @@ class GaoDeUtils { * 获取地址经纬度 */ static def parserByGaode(def city, def address) { - JSONObject json = GaoDeApiUtils.geo(city, address) + Map params = new HashMap() + params.put("key", GAO_DE_AK[keyUsed.intValue()]) + params.put("city", city) + params.put("address", address) + String content + try { + content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) if (json && json.info && json.info == "OK") { def ret = json.geocodes if (ret) { From 2c24a623a7afc98a0389cbfb5d747c507e8d6d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 17:05:10 +0800 Subject: [PATCH 088/118] =?UTF-8?q?=E5=B1=85=E5=A7=94=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/CommitteeGeoHash.groovy | 44 +++++++++++++++++++ .../temp/shenjian/poi/CommitteeLatLng.groovy | 27 ++++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy new file mode 100644 index 000000000..53b0679f0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 国家标准居委会经纬度geohash化 + * + * @author shenjian + * @since 2021/7/22 + */ +class CommitteeGeoHash { + + static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + dealPoi(new BasicDBObject("status_location", 0).append("lng", new BasicDBObject('$ne', null))) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + villagesConn.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + double lng = Double.valueOf(obj.lng) + double lat = Double.valueOf(obj.lat) + def geohash = SeaweedUtils.encodeByGeoHash(lng, lat, 8); + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("geohash", geohash))) + println("已处理小区:" + obj) + } + }) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index cadb279a7..1f5f696cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -22,6 +22,7 @@ class CommitteeLatLng { static void main(String[] args) { dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) + //dealLocation() executorService.shutdown() } @@ -32,13 +33,16 @@ class CommitteeLatLng { @Override void run() { def city = obj.city_name + if (city == "市辖区") { + city = obj.province_name + } def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; def location = GaoDeUtils.parserByGaode(city, address); if (location && location.lng) { - def lng = location.lng - def lat = location.lat + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) - .append("lng", lng).append("lat", lat).append("flag_location", "已获取经纬度"))) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度"))) } else { villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) } @@ -47,4 +51,21 @@ class CommitteeLatLng { }) } } + + static dealLocation() { + villagesConn.find(new BasicDBObject("status_location", 0)).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + if (obj.lng && obj.lat) { + def lng = Double.valueOf(obj.lng) + def lat = Double.valueOf(obj.lat) + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("location", new BasicDBObject("lat", lat).append("lon", lng)))) + } + println("已处理小区:" + obj) + } + }) + } + } } From a940ba26ff267037ec872781c9827b4fa94b14a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 4 Aug 2021 13:07:52 +0800 Subject: [PATCH 089/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=94=BF=E5=BA=9C?= =?UTF-8?q?=E6=9C=BA=E5=85=B3=E3=80=81=E4=B8=A7=E8=91=AC=E8=AE=BE=E6=96=BD?= =?UTF-8?q?=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/dingding/DingTalkUtils.groovy | 2 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 9603351a9..209ab9fe2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,7 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-06-01", "yyyy-MM-dd") ; + Date now = DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 00ee277ea..2535bcf4f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -162,17 +162,20 @@ class SeaweedAroundPoi extends BaseBatch { } public static final Map KEYWORDS_MAP = new HashMap(){{ - put("地铁", "150500"); - put("公交", "1507"); - put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); - put("写字楼", "120201"); - put("学校","141200|141201|141202|141203|141204"); - put("银行","1601|1603"); - put("大型超市","0604"); - put("商场","0601"); - put("餐饮","0501|0502|0503|0504"); - put("旅游景点","110000"); - put("图书馆","140500"); +// put("地铁", "150500"); +// put("公交", "1507"); +// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); +// put("写字楼", "120201"); +// put("学校","141200|141201|141202|141203|141204"); +// put("银行","1601|1603"); +// put("大型超市","0604"); +// put("商场","0601"); +// put("餐饮","0501|0502|0503|0504"); +// put("旅游景点","110000"); +// put("图书馆","140500"); + put("政府机关","130100|130101|130102|130103|130104|130105|130106|130107"); + put("丧葬设施", "071900|071901|071902|071903") + }}; @Override From 230c825cf588cb066d5ed3dfb550428761ad8572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 4 Aug 2021 13:21:26 +0800 Subject: [PATCH 090/118] =?UTF-8?q?=E5=85=B7=E4=BD=93=E7=82=B9=E7=BB=8F?= =?UTF-8?q?=E7=BA=AC=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/poi/GetPoi.groovy | 39 ++++++++++++++ .../seaweed/SeaweedGeoHashExport.groovy | 53 +++++++++++-------- .../temp/shenjian/util/SeaweedUtils.groovy | 3 +- 3 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy new file mode 100644 index 000000000..7ff7cb903 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 获取具体点poi + * + * @author shenjian + * @since 2021/7/26 + */ +class GetPoi { + + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + List pois = GaoDeUtils.searchNearDtoByPoi("无锡","190301",120.411465,31.864912, 1000, "江苏省江阴市周庄镇兴隆北路"); + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy index 154efe43e..a0cede622 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy @@ -1,6 +1,9 @@ package com.fangjia.script.temp.shenjian.seaweed +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject @@ -14,27 +17,35 @@ import com.mongodb.DBObject class SeaweedGeoHashExport { static void main(String[] args) { - List list = new ArrayList<>() + print(SeaweedUtils.encodeByGeoHash(30.8006287800, 108.3835777247, 8)) + //List> data = com.fangjia.script.util.ExcelUtils.read("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx") +/* for (List str : data) { + String geohash = str.get(0) + str.add(SeaweedUtils.decodeByGeoHash(geohash)) + }*/ - DBObject query = new BasicDBObject() - query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁") - - DBObject filter = new BasicDBObject() - filter.put("lng", 1) - filter.put("lat", 1) - DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp -> - final def obj = objTemp - if (obj.lng && obj.lat) { - def lon = obj.lng - def lat = obj.lat - list.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) - } - } - def ret = [] - ret << ['小区geohash'] - for (String val : list) { - ret << [val] - } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); + //com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx", data, "geohash"); +// DBObject query = new BasicDBObject() +// query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁") +// +// DBObject filter = new BasicDBObject() +// filter.put("lng", 1) +// filter.put("lat", 1) +// filter.put("name", 1) +// +// def rows = [] +// DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp -> +// final def obj = objTemp +// if (obj.lng && obj.lat) { +// def lon = obj.lng +// def lat = obj.lat +// def geohash = SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7) +// rows << [lon, lat, obj.name, geohash] +// } +// } +// def ret = [] +// ret << ["经度", "纬度", "小区名", "小区geohash"] +// ret.addAll(rows) +// com.fangjia.script.util.ExcelUtils.write("D:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 9fa4739f8..f21ff7ae7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -198,7 +198,8 @@ class SeaweedUtils { * @param args */ static def decodeByGeoHash(String geohash) { - return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + return DataHandleUtils.getDoubleRound(6, Geohash.decodeLatitude(geohash)) + "," + + DataHandleUtils.getDoubleRound(6, Geohash.decodeLongitude(geohash)) } static void main(String[] args) { From 565acf1eb4e5d6ea4379d150524116c1df366861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 9 Aug 2021 09:10:28 +0800 Subject: [PATCH 091/118] t --- .../fangjia/script/temp/shenjian/Temp.groovy | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 10e63221e..01d0e126d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,50 +1,11 @@ package com.fangjia.script.temp.shenjian -import com.alibaba.fastjson.JSONArray -import com.alibaba.fastjson.JSONObject -import com.baidubce.http.ApiExplorerClient -import com.baidubce.http.AppSigner -import com.baidubce.http.HttpMethodName -import com.baidubce.model.ApiExplorerRequest -import com.baidubce.model.ApiExplorerResponse -import com.fangjia.script.statistics.poi.AmapPoi -import com.fangjia.script.temp.shenjian.util.SeaweedUtils -import org.elasticsearch.geometry.utils.Geohash - /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - String path = "https://goojia.api.bdymkt.com/estimate"; - ApiExplorerRequest request = new ApiExplorerRequest(HttpMethodName.GET, path); - request.setCredentials("fccf809a0f7748449154af926897aecb", "af1f0088a05c493ab20133ba470f87ef "); - // 设置header参数 - request.addHeaderParameter("Content-Type", "application/json; charset=utf-8"); - - // 设置query参数 - request.addQueryParameter("address", "上海宝山呼玛三村"); - request.addQueryParameter("city", "上海"); - request.addQueryParameter("size", "100"); - - - ApiExplorerClient client = new ApiExplorerClient(new AppSigner()); - - try { - ApiExplorerResponse response = client.sendRequest(request); - // 返回结果格式为Json字符串 - System.out.println(response.getResult()); - } catch (Exception e) { - e.printStackTrace(); - } } - -// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 -// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米 -// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 -// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米 -// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米 -// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 } From 362049c9e343e1301916a5b4507747e1dbe29581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 16 Aug 2021 15:48:46 +0800 Subject: [PATCH 092/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E7=BB=8F=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SchoolLatLng.groovy | 65 +++++++++++++++++++ .../temp/shenjian/util/ExcelUtils.groovy | 28 ++++---- 2 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy new file mode 100644 index 000000000..219dd491c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 学校经纬度获取 + * + * @author shenjian + * @since 2021/8/16 + */ +class SchoolLatLng { + + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static String name = "大连市义务教育学校名录test"; + + static void main(String[] args) { + List> schools = ExcelUtils.read(new FileInputStream("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + ".xlsx")); + Map> result = new ConcurrentHashMap<>(); + CountDownLatch latch = new CountDownLatch(schools.size()); + + for (List schoolTemp : schools) { + final List school = schoolTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + def city = "大连" + def list = new ArrayList(); + def address = school.get(4) + def location = GaoDeUtils.parserByGaode(city, address); + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + list.add(lng) + list.add(lat) + result.put(address, list) + } + println("已处理:" + address) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + + List> head = new ArrayList<>() + for (List school : schools) { + def address = school.get(4) + school.addAll(result.get(address)) + } + EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("小学").doWrite(schools) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy index 229499b80..fa0927eee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -13,36 +13,36 @@ import com.alibaba.excel.event.AnalysisEventListener */ class ExcelUtils { - static List> read(String filePath) { - SimpleExcelListener listener = new SimpleExcelListener(filePath); - ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); - excelReader.readAll() + public static List> read(InputStream inputStream) { + SimpleExcelListener listener = new SimpleExcelListener(); + ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener) + .headRowNumber(1).build(); + excelReader.readAll(); List> dataList = listener.getDataList(); - excelReader.finish() + excelReader.finish(); return dataList; } - class SimpleExcelListener extends AnalysisEventListener { - - private List> dataList = new ArrayList<>() + public static class SimpleExcelListener extends AnalysisEventListener { + private List> dataList = new ArrayList<>(); @Override - void invoke(Object o, AnalysisContext analysisContext) { - Map map = o; - dataList.add(new ArrayList(map.values())) + public void invoke(Object o, AnalysisContext analysisContext) { + Map map = (Map) o; + dataList.add(new ArrayList<>(map.values())); } @Override - void doAfterAllAnalysed(AnalysisContext analysisContext) { + public void doAfterAllAnalysed(AnalysisContext analysisContext) { } List> getDataList() { - return dataList + return dataList; } void setDataList(List> dataList) { - this.dataList = dataList + this.dataList = dataList; } } } From daf23e52c5d7b346a2b33f7134e834b30ffedb8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 16 Aug 2021 16:18:24 +0800 Subject: [PATCH 093/118] =?UTF-8?q?easyexcel=E4=BE=9D=E8=B5=96poi=E7=89=88?= =?UTF-8?q?=E6=9C=AC3.17?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 871548bdf..2e06fc2a1 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -366,12 +366,12 @@ org.apache.poi poi - 3.10-FINAL + 3.17 org.apache.poi poi-ooxml - 3.10-FINAL + 3.17 xml-apis From 347ef3f4db3728a71c86a4c4e40cfc128f537b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 18 Aug 2021 14:09:39 +0800 Subject: [PATCH 094/118] =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SchoolLatLng.groovy | 31 ++++++---- .../temp/shenjian/util/ExcelUtils.groovy | 56 +++++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 56 +++++++++++++++++++ 3 files changed, 132 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy index 219dd491c..08dc2c38d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.excel.EasyExcel +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils @@ -34,14 +36,19 @@ class SchoolLatLng { try { def city = "大连" def list = new ArrayList(); - def address = school.get(4) - def location = GaoDeUtils.parserByGaode(city, address); - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - list.add(lng) - list.add(lat) - result.put(address, list) + def address = school.get(1) + def pois = GaoDeUtils.searchPoi(city, address); + if (CollectionUtils.isNotEmpty(pois)) { + for (AmapPoi amapPoi : pois) { + if (address.contains(amapPoi.getName())) { + String[] locations = amapPoi.getLocation().split(",") + list.add(locations[0]) + list.add(locations[1]) + list.add(amapPoi.getId()) + result.put(address, list) + break; + } + } } println("已处理:" + address) } catch(Exception ex) { @@ -56,10 +63,12 @@ class SchoolLatLng { List> head = new ArrayList<>() for (List school : schools) { - def address = school.get(4) - school.addAll(result.get(address)) + def address = school.get(1) + if (result.contains(address)) { + school.addAll(result.get(address)) + } } - EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("小学").doWrite(schools) + EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("学校").doWrite(schools) executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy index fa0927eee..639cd37a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -1,9 +1,13 @@ package com.fangjia.script.temp.shenjian.util +import com.alibaba.excel.EasyExcel import com.alibaba.excel.EasyExcelFactory import com.alibaba.excel.ExcelReader import com.alibaba.excel.context.AnalysisContext import com.alibaba.excel.event.AnalysisEventListener +import com.alibaba.excel.read.metadata.ReadSheet + +import java.util.concurrent.ConcurrentHashMap /** * excel工具类 @@ -13,6 +17,45 @@ import com.alibaba.excel.event.AnalysisEventListener */ class ExcelUtils { + + static List> read(String filePath) { + SimpleExcelListener listener = new SimpleExcelListener(filePath); + ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); + excelReader.readAll() + List> dataList = listener.getDataList(); + excelReader.finish() + return dataList; + } + + static List> readSheet(String fileName, String[] sheetNames) { + Map>> result = new ConcurrentHashMap<>(); + ExcelReader excelReader = EasyExcel.read(fileName).build(); + + // 每个sheet分别创建监听器 + SimpleExcelListener listener0 = new SimpleExcelListener(); + SimpleExcelListener listener1 = new SimpleExcelListener(); + + ReadSheet readSheet0 = EasyExcel.readSheet(0).registerReadListener(listener0).build(); + + ReadSheet readSheet1 = EasyExcel.readSheet(1).registerReadListener(listener1).build(); + + // 这里注意 一定要把sheet1 sheet2 一起传进去,不然有个问题就是03版的excel 会读取多次,浪费性能 + // 可以传多个ReadSheet对象,读多个sheet + excelReader.read(readSheet0, readSheet1); + + // 这里千万别忘记关闭,读的时候会创建临时文件,到时磁盘会崩的 + excelReader.finish(); + + ArrayList listenerList = new ArrayList<>(); + listenerList.add(listener0); + listenerList.add(listener1); + + for (SimpleExcelListener listener : listenerList) { + result.put(listener.getSheetName(), listener.getDataList()) + } + return result + } + public static List> read(InputStream inputStream) { SimpleExcelListener listener = new SimpleExcelListener(); ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener) @@ -25,6 +68,15 @@ class ExcelUtils { public static class SimpleExcelListener extends AnalysisEventListener { private List> dataList = new ArrayList<>(); + private String sheetName; + + public SimpleExcelListener() { + + } + + public SimpleExcelListener(String sheetName) { + this.sheetName = sheetName + } @Override public void invoke(Object o, AnalysisContext analysisContext) { @@ -44,5 +96,9 @@ class ExcelUtils { void setDataList(List> dataList) { this.dataList = dataList; } + + public String getSheetName() { + return sheetName + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 4d7cee9f2..1c0188047 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -165,6 +165,62 @@ class GaoDeUtils { return result } + /** + * 查找关键字经纬度 + */ + static List searchPoi(String city, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.keywords = keywords + params.city = city + params.put("extensions", "all") + params.put("citylimit","true") + params.put("offset","20") + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } else { + break + } + } + return result + } + /** * 获取经纬度之间距离 */ From ca55e1a30e91be3e991f9d59ad607957b2297e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 2 Sep 2021 19:08:26 +0800 Subject: [PATCH 095/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=B5=A6=E4=B8=9C?= =?UTF-8?q?=E5=AF=B9=E5=8F=A3=E5=B0=8F=E5=AD=A6=E5=88=9D=E4=B8=AD=E6=8A=93?= =?UTF-8?q?=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/SeaweedSchool.groovy | 159 +++++++++++ .../script/statistics/json/housedetail.json | 42 +++ .../script/statistics/json/houselist.json | 255 ++++++++++++++++++ .../script/statistics/json/school.json | 65 +++++ .../statistics/seaweed/HouseDetail.java | 79 ++++++ .../script/statistics/seaweed/HouseList.java | 107 ++++++++ .../script/statistics/seaweed/School.java | 85 ++++++ 7 files changed, 792 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy new file mode 100644 index 000000000..09131cfb4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -0,0 +1,159 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.Constants +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.MongoCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.jdom.Content; + +/** + * @author shenjian* @since 2021/9/2 + */ +class SeaweedSchool { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.mapmapping.com") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("Content-Type", "application/json") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Referer", "https://servicewechat.com/wx5568ec264e167aeb/9/page-frame.html") + headers.put("Accept-Encoding", "gzip, deflate, br") + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + final String area = "浦东新区"; + final int perPage = 10 + final int total = 4733; + for (int page = 1; page < total / perPage; page++) { + Thread.sleep(10000) + print("当前页:" + page) + try { + Map params = new HashMap(); + params.put("city", "上海市") + params.put("area", area) + params.put("page", String.valueOf(page)) + params.put("perPage", String.valueOf(perPage)) + params.put("priceType", "") + params.put("order", "") + params.put("source", "1") + str = re.get("https://mapmapping.com/school-web/house/getHouseInfoList", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + println str + if (StringUtils.isNotEmpty(str)) { + HouseList houseList = JSONObject.parseObject(str, HouseList.class); + if (houseList != null && "000".equals(houseList.getCode())) { + /*Integer total = houseList.getContent().getTotal();*/ + List listDTOS = houseList.getContent().getList(); + if (CollectionUtils.isNotEmpty(listDTOS)) { + for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { + Thread.sleep(5000) + Integer id = listDTO.getId(); + //# 房屋详情 + HouseDetail houseDetail = getHouseDetail(re, id, headers) + //# 对口小学 + School primarySchool = getPrimarySchool(re, id, headers) + //# 对口中学 + School middleSchool = getMiddleSchool(re, id, headers) + String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) + saveData(houseDetail, primarySchool, middleSchool, url); + } + } + } + } + } + } + + private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) { + DBObject dbObject = new BasicDBObject() + dbObject.put("url", url) + dbObject.put("root", "沪上学区房") + dbObject.put("c_date", new Date()) + dbObject.put("city", "上海") + dbObject.put("region", houseDetail.getContent().getArea()) + dbObject.put("name", houseDetail.getContent().getName()) + dbObject.put("address", houseDetail.getContent().getAddress()) + List primarySchools = primarySchool.getContent(); + if (CollectionUtils.isNotEmpty(primarySchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : primarySchools) { + DBObject schoolObject = new BasicDBObject(); + if ("小学".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("primarySchool", schools) + } + + List middleSchools = middleSchool.getContent(); + if (CollectionUtils.isNotEmpty(middleSchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : middleSchools) { + DBObject schoolObject = new BasicDBObject(); + if ("初中".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("middleSchool", schools) + } + + + collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) + } + + private static HouseDetail getHouseDetail(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getHouseInfoDetail", params, headers, "UTF-8"); + return JSONObject.parseObject(str, HouseDetail.class) + } + + private static School getPrimarySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "2") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + + private static School getMiddleSchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "3") + params.put("page", "1") + params.put("perPage", "4") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json new file mode 100644 index 000000000..64af63aad --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json @@ -0,0 +1,42 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "id": 59151, + "name": "凌高小区", + "imgsUrl": "[\"https://image1.ljcdn.com/hdic-resblock/5f42d5a7-acd9-4a6f-a9f9-92dd651adf6c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ff4c17ac-759a-4e52-bf24-561fa02e851a.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e1ace69b-ce4b-43f9-9e5d-b14244d510cc.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/359a3b16-9485-49ef-b1eb-2cae27751622.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0a17322f-4cdd-43b0-aec0-2d6114a5d636.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/174aac59-4da4-4e88-ad6c-d76a17a59e17.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a6c09e7a-8e96-48c3-b291-20603d61355f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ac1cb228-197a-4e60-b0b0-1bc1c451d00e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e4e9ec1b-5430-4269-b5f7-eb934e189e85.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d9e874a1-df15-42de-b5e7-fb369db97723.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e620cc27-d24b-4cf1-ab4e-d6b3467c180d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b3c00e2e-bacc-4683-abab-8da0a6395972.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b77c7777-41cd-44f0-969f-7d7083d402fe.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/712a3c9a-0d43-410b-becb-ebe47e62fc6d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c44cee38-afbe-42b3-bdf7-f3875a5d0837.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/288933d6-5f8c-4510-8396-15acd8f4580c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c1611809-a633-470c-871c-3a2862e50c79.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/07096876-dd38-415a-ac95-27b79452f664.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae7ffa23-426e-42b7-a509-1a6752c95b72.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0fa04835-9077-4ab6-9e24-72955db66906.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/eb7e289a-8e03-4067-81fa-c633d1a429a5.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/deeade88-3b91-4702-9fdf-68b556ec9921.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/75270a55-bef0-4274-8275-825abae9dce3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0dec445b-f47f-4f2f-996f-c77915d52673.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f05af36d-af7e-4d3f-a8ea-8f23536d2d23.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/40fa734e-682d-4ef6-8094-37e5d281a75f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a3e582a6-19d0-4eeb-9735-2a5a7916187e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f56baa4b-d531-4782-a619-9b22a44b8663.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cc14001f-08ce-4769-9c14-ef4ce1e1f406.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/1153dabe-4ecc-4246-aad2-23ac4575f67f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0712ae4-63dc-477f-9335-1e7147c201d1.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4051310b-ee71-489c-af36-dbb2dc8dd4e0.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/272b80dd-5356-4e7a-beaa-c9d2908d6214.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c238f4da-acc3-4abd-bd62-4c43d59aa46b.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae234e8e-d1d7-4a96-ba27-a333364ae907.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7dec0fa6-508b-4220-b20b-77560fe72c03.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/70cbd711-3bd8-4e6b-9b5e-a13ca6b41cfb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6a258444-96ee-4200-820a-449bb611c801.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0983610c-8ead-472c-a70b-9c50ede9cfd8.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/461dbab6-9cc2-4436-a466-3a06fb1bfe1e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7df6667a-948c-435b-81cb-c1c798614b2e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/97feda14-4672-481f-b127-c28b185cd0ca.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a131d9ca-efd3-4609-a43c-d32978d99cf3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/efd8958c-2108-4660-b8c1-611dad53abf2.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/93c57a6a-0832-4ea0-8512-fcf0d9f5d751.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0dbd942-0ce3-45e9-b1f9-f0af683db82e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b492aeef-172e-40ab-b849-ab2c633d9d13.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/87bde5bd-762b-4b47-ad13-9c1e4e083491.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cb526f64-6bc2-4960-bd71-d650e55123f7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/8b18255e-732e-425d-819b-a753a6191174.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6e45cfc3-cff9-4f47-ac1a-6d9525e2c883.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0463904e-94a1-4720-af81-08c5b289cbeb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/2e518748-1f82-41d1-8bd1-0628830456a7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ad79921b-4aac-4f54-bac9-59a624c95be9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/046e791a-e361-4383-8b6c-609fb1b6ca42.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b219aa06-e7bc-4649-ba96-03996a1565d9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d2e5e0da-7722-4f96-ad68-07e83bf3de5d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4ccfea7c-6c44-4f6f-a900-3aaa5354c356.jpg.710x400.jpg\"]", + "area": "浦东", + "town": "洋泾", + "price": "6.64w", + "priceDesc": "二手挂牌均价", + "plotType": "塔楼", + "leaseNum": "4", + "sellNum": "11", + "dealNum": "0", + "buildTime": "1993年建成", + "houseNum": "1344户", + "building": "6栋", + "propertyFee": "0.75元/平米/月", + "propertyCompany": "上海益镇物业管理有限公司", + "developers": "上海建工房产有限公司", + "lat": 31.247013798320733, + "lng": 121.54954132423184, + "imgArr": [ + "https://img.mapmapping.com/b9bb10e4-d216-4fe3-ac8d-0f3d778e288b.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/28582f65-a02a-47df-b730-9f7f5a5851ee.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/5997bbc0-2322-4d93-9714-2d4513db7677.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/4cd79c3a-f543-4a94-89e8-e80241ff7876.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/3b517fea-b3d3-4b1e-8f58-50e4775d7cfb.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/489d39ee-73ee-4d47-9f76-151c4313d5c3.png?imageView2/2/w/700/h/500/q/75" + ], + "focus": false, + "tags": [ + "浦东", + "洋泾", + "塔楼" + ], + "view": 15312, + "address": "上海市浦东新区栖山路401弄-11号-临" + }, + "timestamp": 1630576612732 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json new file mode 100644 index 000000000..9946d0550 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json @@ -0,0 +1,255 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "total": 4733, + "list": [ + { + "id": 59151, + "name": "凌高小区", + "area": "浦东", + "img": "https://img.mapmapping.com/a68e7f60-a8d8-4e25-ade8-2d77c86562cd.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "11", + "leaseNum": "4", + "price": "6.64w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.25306680262166000", + "lng": "121.55601323149715000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.53w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58324, + "name": "梅园四街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/775871a1-e5a9-4e01-88ed-a39f040b51ac.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "15", + "leaseNum": "11", + "price": "13.42w", + "formatPrice": null, + "year": "1988年建成", + "lat": "31.23754371649258600", + "lng": "121.53281691712684000", + "tags": [ + "近地铁2号线东昌路站" + ], + "plotType": "塔楼/板楼/塔板结合", + "buildTime": "1988年建成", + "view": "1.52w", + "tag": "近地铁2号线东昌路站" + }, + { + "id": 59165, + "name": "中星海上景庭", + "area": "浦东", + "img": "https://img.mapmapping.com/4f9d8c2a-8b89-49a8-b45f-ecaff4f3f588.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "8", + "leaseNum": "3", + "price": "10.86w", + "formatPrice": null, + "year": "2004年建成", + "lat": "31.24523893361267700", + "lng": "121.55501162085352000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "板楼", + "buildTime": "2004年建成", + "view": "1.50w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58477, + "name": "海怡别墅(公寓)", + "area": "浦东", + "img": "https://img.mapmapping.com/f50ed86d-474f-47c5-8f83-7c203c510d92.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "0", + "leaseNum": "0", + "price": "7.32w", + "formatPrice": null, + "year": "1994年建成", + "lat": "31.23963538757626600", + "lng": "121.53779361948267000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1994年建成", + "view": "1.50w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 59148, + "name": "崮山小区一街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/b651bbba-97ba-4ce0-831c-925e80b4459f.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范附属小学羽山校区", + "sellerNum": "10", + "leaseNum": "5", + "price": "11.81w", + "formatPrice": null, + "year": "1999年建成", + "lat": "31.24282189657667000", + "lng": "121.56992501943853000", + "tags": [ + "近地铁9号线芳甸路站" + ], + "plotType": "板楼", + "buildTime": "1999年建成", + "view": "1.50w", + "tag": "近地铁9号线芳甸路站" + }, + { + "id": 55895, + "name": "证大家园(四期)", + "area": "浦东", + "img": "https://img.mapmapping.com/d7d0671f-b0ed-4082-a9f6-1883eca3ac85.png?imageView2/2/w/240/h/200/q/75", + "town": "高行", + "schoolTag": "浦东新区福山证大外国语小学", + "sellerNum": "11", + "leaseNum": "7", + "price": "9.63w", + "formatPrice": null, + "year": "2006年建成", + "lat": "31.28831283871331500", + "lng": "121.60787797808233000", + "tags": [ + "近地铁6号线巨峰路站" + ], + "plotType": "板楼", + "buildTime": "2006年建成", + "view": "1.49w", + "tag": "近地铁6号线巨峰路站" + }, + { + "id": 58709, + "name": "新光大楼", + "area": "浦东", + "img": "https://img.mapmapping.com/f8e1dffb-fa85-457b-8a97-fb019239bdf6.png?imageView2/2/w/240/h/200/q/75", + "town": "潍坊", + "schoolTag": "浦东新区明珠小学A区", + "sellerNum": "3", + "leaseNum": "2", + "price": "11.95w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.23234474870607700", + "lng": "121.52462643697852000", + "tags": [ + "近地铁9号线商城路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.49w", + "tag": "近地铁9号线商城路站" + }, + { + "id": 59322, + "name": "盛世年华", + "area": "浦东", + "img": "https://img.mapmapping.com/83727155-29f5-47d9-b54d-4042d9e625ec.png?imageView2/2/w/240/h/200/q/75", + "town": "源深", + "schoolTag": "浦东新区福山花园外国语小学", + "sellerNum": "4", + "leaseNum": "11", + "price": "14.94w", + "formatPrice": null, + "year": "2005年建成", + "lat": "31.23494392135101000", + "lng": "121.54545194344688000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "2005年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58326, + "name": "梅园七街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/30cf630f-830b-4e43-8dc5-648bb7dfb760.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "2", + "leaseNum": "2", + "price": "11.13w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24378516838488700", + "lng": "121.53455423998314000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58599, + "name": "陈家门小区", + "area": "浦东", + "img": "https://img.mapmapping.com/7a9f43ef-67c4-4168-bf24-0190689d331d.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区梅园小学", + "sellerNum": "3", + "leaseNum": "1", + "price": "9.27w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24726096884606800", + "lng": "121.53135690118148000", + "tags": [ + "近地铁4号线浦东大道站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁4号线浦东大道站" + } + ], + "pageNum": 1, + "pageSize": 10, + "size": 10, + "startRow": 1, + "endRow": 10, + "pages": 474, + "prePage": 0, + "nextPage": 2, + "isFirstPage": true, + "isLastPage": false, + "hasPreviousPage": false, + "hasNextPage": true, + "navigatePages": 8, + "navigatepageNums": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ], + "navigateFirstPage": 1, + "navigateLastPage": 8 + }, + "timestamp": 1630575199849 +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json new file mode 100644 index 000000000..125089046 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json @@ -0,0 +1,65 @@ +{ + "code": "000", + "desc": "操作成功", + "content": [ + { + "id": 272113, + "name": "第六师范学校第二附属小学西校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "小学", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "41", + "sellNum": "188", + "plotPrice": "8.69", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + }, + { + "id": 273287, + "name": "进才中学北校苗圃路校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "初中", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "85", + "sellNum": "339", + "plotPrice": "9.33", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + } + ], + "timestamp": 1630576702404 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java new file mode 100644 index 000000000..b02953be7 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java @@ -0,0 +1,79 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseDetail { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "imgsUrl") + private String imgsUrl; + @JSONField(name = "area") + private String area; + @JSONField(name = "town") + private String town; + @JSONField(name = "price") + private String price; + @JSONField(name = "priceDesc") + private String priceDesc; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "dealNum") + private String dealNum; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "houseNum") + private String houseNum; + @JSONField(name = "building") + private String building; + @JSONField(name = "propertyFee") + private String propertyFee; + @JSONField(name = "propertyCompany") + private String propertyCompany; + @JSONField(name = "developers") + private String developers; + @JSONField(name = "lat") + private Double lat; + @JSONField(name = "lng") + private Double lng; + @JSONField(name = "imgArr") + private List imgArr; + @JSONField(name = "focus") + private Boolean focus; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "view") + private Integer view; + @JSONField(name = "address") + private String address; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java new file mode 100644 index 000000000..cd49ae026 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java @@ -0,0 +1,107 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseList { + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "total") + private Integer total; + @JSONField(name = "list") + private List list; + @JSONField(name = "pageNum") + private Integer pageNum; + @JSONField(name = "pageSize") + private Integer pageSize; + @JSONField(name = "size") + private Integer size; + @JSONField(name = "startRow") + private Integer startRow; + @JSONField(name = "endRow") + private Integer endRow; + @JSONField(name = "pages") + private Integer pages; + @JSONField(name = "prePage") + private Integer prePage; + @JSONField(name = "nextPage") + private Integer nextPage; + @JSONField(name = "isFirstPage") + private Boolean isFirstPage; + @JSONField(name = "isLastPage") + private Boolean isLastPage; + @JSONField(name = "hasPreviousPage") + private Boolean hasPreviousPage; + @JSONField(name = "hasNextPage") + private Boolean hasNextPage; + @JSONField(name = "navigatePages") + private Integer navigatePages; + @JSONField(name = "navigatepageNums") + private List navigatepageNums; + @JSONField(name = "navigateFirstPage") + private Integer navigateFirstPage; + @JSONField(name = "navigateLastPage") + private Integer navigateLastPage; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "area") + private String area; + @JSONField(name = "img") + private String img; + @JSONField(name = "town") + private String town; + @JSONField(name = "schoolTag") + private String schoolTag; + @JSONField(name = "sellerNum") + private String sellerNum; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "price") + private String price; + @JSONField(name = "formatPrice") + private Object formatPrice; + @JSONField(name = "year") + private String year; + @JSONField(name = "lat") + private String lat; + @JSONField(name = "lng") + private String lng; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "view") + private String view; + @JSONField(name = "tag") + private String tag; + } + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java new file mode 100644 index 000000000..1beb5153b --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -0,0 +1,85 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class School { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private List content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "nature") + private Object nature; + @JSONField(name = "city") + private String city; + @JSONField(name = "area") + private String area; + @JSONField(name = "type") + private String type; + @JSONField(name = "quality") + private String quality; + @JSONField(name = "location") + private Object location; + @JSONField(name = "picture") + private String picture; + @JSONField(name = "phone") + private String phone; + @JSONField(name = "distanceNum") + private Object distanceNum; + @JSONField(name = "plotNum") + private String plotNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "plotPrice") + private String plotPrice; + @JSONField(name = "address") + private Object address; + @JSONField(name = "lat") + private Object lat; + @JSONField(name = "lng") + private Object lng; + @JSONField(name = "scope") + private Object scope; + @JSONField(name = "score") + private Object score; + @JSONField(name = "scopeArr") + private Object scopeArr; + @JSONField(name = "scopeTag") + private Object scopeTag; + @JSONField(name = "tag") + private Object tag; + @JSONField(name = "shortName") + private Object shortName; + @JSONField(name = "view") + private Object view; + @JSONField(name = "construct") + private Object construct; + @JSONField(name = "echelon") + private Object echelon; + @JSONField(name = "town") + private Object town; + } +} From 5f37c826743b7ae13ec1cd88b153bcd1989dc6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 2 Sep 2021 19:08:26 +0800 Subject: [PATCH 096/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=B5=A6=E4=B8=9C?= =?UTF-8?q?=E5=AF=B9=E5=8F=A3=E5=B0=8F=E5=AD=A6=E5=88=9D=E4=B8=AD=E6=8A=93?= =?UTF-8?q?=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/SeaweedSchool.groovy | 159 +++++++++++ .../script/statistics/json/housedetail.json | 42 +++ .../script/statistics/json/houselist.json | 255 ++++++++++++++++++ .../script/statistics/json/school.json | 65 +++++ .../statistics/seaweed/HouseDetail.java | 79 ++++++ .../script/statistics/seaweed/HouseList.java | 107 ++++++++ .../script/statistics/seaweed/School.java | 85 ++++++ 7 files changed, 792 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy new file mode 100644 index 000000000..5981472a2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -0,0 +1,159 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.Constants +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.MongoCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.jdom.Content; + +/** + * @author shenjian* @since 2021/9/2 + */ +class SeaweedSchool { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.mapmapping.com") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("Content-Type", "application/json") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Referer", "https://servicewechat.com/wx5568ec264e167aeb/9/page-frame.html") + headers.put("Accept-Encoding", "gzip, deflate, br") + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + final String area = "浦东新区"; + final int perPage = 10 + final int total = 4733; + for (int page = 15; page < total / perPage; page++) { + Thread.sleep(1000) + print("当前页:" + page) + try { + Map params = new HashMap(); + params.put("city", "上海市") + params.put("area", area) + params.put("page", String.valueOf(page)) + params.put("perPage", String.valueOf(perPage)) + params.put("priceType", "") + params.put("order", "") + params.put("source", "1") + str = re.get("https://mapmapping.com/school-web/house/getHouseInfoList", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + println str + if (StringUtils.isNotEmpty(str)) { + HouseList houseList = JSONObject.parseObject(str, HouseList.class); + if (houseList != null && "000".equals(houseList.getCode())) { + /*Integer total = houseList.getContent().getTotal();*/ + List listDTOS = houseList.getContent().getList(); + if (CollectionUtils.isNotEmpty(listDTOS)) { + for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { + Thread.sleep(1000) + Integer id = listDTO.getId(); + //# 房屋详情 + HouseDetail houseDetail = getHouseDetail(re, id, headers) + //# 对口小学 + School primarySchool = getPrimarySchool(re, id, headers) + //# 对口中学 + School middleSchool = getMiddleSchool(re, id, headers) + String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) + saveData(houseDetail, primarySchool, middleSchool, url); + } + } + } + } + } + } + + private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) { + DBObject dbObject = new BasicDBObject() + dbObject.put("url", url) + dbObject.put("root", "沪上学区房") + dbObject.put("c_date", new Date()) + dbObject.put("city", "上海") + dbObject.put("region", houseDetail.getContent().getArea()) + dbObject.put("name", houseDetail.getContent().getName()) + dbObject.put("address", houseDetail.getContent().getAddress()) + List primarySchools = primarySchool.getContent(); + if (CollectionUtils.isNotEmpty(primarySchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : primarySchools) { + DBObject schoolObject = new BasicDBObject(); + if ("小学".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("primarySchool", schools) + } + + List middleSchools = middleSchool.getContent(); + if (CollectionUtils.isNotEmpty(middleSchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : middleSchools) { + DBObject schoolObject = new BasicDBObject(); + if ("初中".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("middleSchool", schools) + } + + + collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) + } + + private static HouseDetail getHouseDetail(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getHouseInfoDetail", params, headers, "UTF-8"); + return JSONObject.parseObject(str, HouseDetail.class) + } + + private static School getPrimarySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "2") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + + private static School getMiddleSchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "3") + params.put("page", "1") + params.put("perPage", "4") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json new file mode 100644 index 000000000..64af63aad --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json @@ -0,0 +1,42 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "id": 59151, + "name": "凌高小区", + "imgsUrl": "[\"https://image1.ljcdn.com/hdic-resblock/5f42d5a7-acd9-4a6f-a9f9-92dd651adf6c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ff4c17ac-759a-4e52-bf24-561fa02e851a.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e1ace69b-ce4b-43f9-9e5d-b14244d510cc.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/359a3b16-9485-49ef-b1eb-2cae27751622.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0a17322f-4cdd-43b0-aec0-2d6114a5d636.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/174aac59-4da4-4e88-ad6c-d76a17a59e17.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a6c09e7a-8e96-48c3-b291-20603d61355f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ac1cb228-197a-4e60-b0b0-1bc1c451d00e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e4e9ec1b-5430-4269-b5f7-eb934e189e85.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d9e874a1-df15-42de-b5e7-fb369db97723.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e620cc27-d24b-4cf1-ab4e-d6b3467c180d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b3c00e2e-bacc-4683-abab-8da0a6395972.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b77c7777-41cd-44f0-969f-7d7083d402fe.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/712a3c9a-0d43-410b-becb-ebe47e62fc6d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c44cee38-afbe-42b3-bdf7-f3875a5d0837.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/288933d6-5f8c-4510-8396-15acd8f4580c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c1611809-a633-470c-871c-3a2862e50c79.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/07096876-dd38-415a-ac95-27b79452f664.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae7ffa23-426e-42b7-a509-1a6752c95b72.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0fa04835-9077-4ab6-9e24-72955db66906.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/eb7e289a-8e03-4067-81fa-c633d1a429a5.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/deeade88-3b91-4702-9fdf-68b556ec9921.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/75270a55-bef0-4274-8275-825abae9dce3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0dec445b-f47f-4f2f-996f-c77915d52673.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f05af36d-af7e-4d3f-a8ea-8f23536d2d23.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/40fa734e-682d-4ef6-8094-37e5d281a75f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a3e582a6-19d0-4eeb-9735-2a5a7916187e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f56baa4b-d531-4782-a619-9b22a44b8663.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cc14001f-08ce-4769-9c14-ef4ce1e1f406.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/1153dabe-4ecc-4246-aad2-23ac4575f67f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0712ae4-63dc-477f-9335-1e7147c201d1.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4051310b-ee71-489c-af36-dbb2dc8dd4e0.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/272b80dd-5356-4e7a-beaa-c9d2908d6214.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c238f4da-acc3-4abd-bd62-4c43d59aa46b.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae234e8e-d1d7-4a96-ba27-a333364ae907.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7dec0fa6-508b-4220-b20b-77560fe72c03.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/70cbd711-3bd8-4e6b-9b5e-a13ca6b41cfb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6a258444-96ee-4200-820a-449bb611c801.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0983610c-8ead-472c-a70b-9c50ede9cfd8.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/461dbab6-9cc2-4436-a466-3a06fb1bfe1e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7df6667a-948c-435b-81cb-c1c798614b2e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/97feda14-4672-481f-b127-c28b185cd0ca.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a131d9ca-efd3-4609-a43c-d32978d99cf3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/efd8958c-2108-4660-b8c1-611dad53abf2.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/93c57a6a-0832-4ea0-8512-fcf0d9f5d751.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0dbd942-0ce3-45e9-b1f9-f0af683db82e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b492aeef-172e-40ab-b849-ab2c633d9d13.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/87bde5bd-762b-4b47-ad13-9c1e4e083491.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cb526f64-6bc2-4960-bd71-d650e55123f7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/8b18255e-732e-425d-819b-a753a6191174.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6e45cfc3-cff9-4f47-ac1a-6d9525e2c883.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0463904e-94a1-4720-af81-08c5b289cbeb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/2e518748-1f82-41d1-8bd1-0628830456a7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ad79921b-4aac-4f54-bac9-59a624c95be9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/046e791a-e361-4383-8b6c-609fb1b6ca42.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b219aa06-e7bc-4649-ba96-03996a1565d9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d2e5e0da-7722-4f96-ad68-07e83bf3de5d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4ccfea7c-6c44-4f6f-a900-3aaa5354c356.jpg.710x400.jpg\"]", + "area": "浦东", + "town": "洋泾", + "price": "6.64w", + "priceDesc": "二手挂牌均价", + "plotType": "塔楼", + "leaseNum": "4", + "sellNum": "11", + "dealNum": "0", + "buildTime": "1993年建成", + "houseNum": "1344户", + "building": "6栋", + "propertyFee": "0.75元/平米/月", + "propertyCompany": "上海益镇物业管理有限公司", + "developers": "上海建工房产有限公司", + "lat": 31.247013798320733, + "lng": 121.54954132423184, + "imgArr": [ + "https://img.mapmapping.com/b9bb10e4-d216-4fe3-ac8d-0f3d778e288b.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/28582f65-a02a-47df-b730-9f7f5a5851ee.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/5997bbc0-2322-4d93-9714-2d4513db7677.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/4cd79c3a-f543-4a94-89e8-e80241ff7876.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/3b517fea-b3d3-4b1e-8f58-50e4775d7cfb.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/489d39ee-73ee-4d47-9f76-151c4313d5c3.png?imageView2/2/w/700/h/500/q/75" + ], + "focus": false, + "tags": [ + "浦东", + "洋泾", + "塔楼" + ], + "view": 15312, + "address": "上海市浦东新区栖山路401弄-11号-临" + }, + "timestamp": 1630576612732 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json new file mode 100644 index 000000000..9946d0550 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json @@ -0,0 +1,255 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "total": 4733, + "list": [ + { + "id": 59151, + "name": "凌高小区", + "area": "浦东", + "img": "https://img.mapmapping.com/a68e7f60-a8d8-4e25-ade8-2d77c86562cd.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "11", + "leaseNum": "4", + "price": "6.64w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.25306680262166000", + "lng": "121.55601323149715000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.53w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58324, + "name": "梅园四街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/775871a1-e5a9-4e01-88ed-a39f040b51ac.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "15", + "leaseNum": "11", + "price": "13.42w", + "formatPrice": null, + "year": "1988年建成", + "lat": "31.23754371649258600", + "lng": "121.53281691712684000", + "tags": [ + "近地铁2号线东昌路站" + ], + "plotType": "塔楼/板楼/塔板结合", + "buildTime": "1988年建成", + "view": "1.52w", + "tag": "近地铁2号线东昌路站" + }, + { + "id": 59165, + "name": "中星海上景庭", + "area": "浦东", + "img": "https://img.mapmapping.com/4f9d8c2a-8b89-49a8-b45f-ecaff4f3f588.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "8", + "leaseNum": "3", + "price": "10.86w", + "formatPrice": null, + "year": "2004年建成", + "lat": "31.24523893361267700", + "lng": "121.55501162085352000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "板楼", + "buildTime": "2004年建成", + "view": "1.50w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58477, + "name": "海怡别墅(公寓)", + "area": "浦东", + "img": "https://img.mapmapping.com/f50ed86d-474f-47c5-8f83-7c203c510d92.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "0", + "leaseNum": "0", + "price": "7.32w", + "formatPrice": null, + "year": "1994年建成", + "lat": "31.23963538757626600", + "lng": "121.53779361948267000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1994年建成", + "view": "1.50w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 59148, + "name": "崮山小区一街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/b651bbba-97ba-4ce0-831c-925e80b4459f.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范附属小学羽山校区", + "sellerNum": "10", + "leaseNum": "5", + "price": "11.81w", + "formatPrice": null, + "year": "1999年建成", + "lat": "31.24282189657667000", + "lng": "121.56992501943853000", + "tags": [ + "近地铁9号线芳甸路站" + ], + "plotType": "板楼", + "buildTime": "1999年建成", + "view": "1.50w", + "tag": "近地铁9号线芳甸路站" + }, + { + "id": 55895, + "name": "证大家园(四期)", + "area": "浦东", + "img": "https://img.mapmapping.com/d7d0671f-b0ed-4082-a9f6-1883eca3ac85.png?imageView2/2/w/240/h/200/q/75", + "town": "高行", + "schoolTag": "浦东新区福山证大外国语小学", + "sellerNum": "11", + "leaseNum": "7", + "price": "9.63w", + "formatPrice": null, + "year": "2006年建成", + "lat": "31.28831283871331500", + "lng": "121.60787797808233000", + "tags": [ + "近地铁6号线巨峰路站" + ], + "plotType": "板楼", + "buildTime": "2006年建成", + "view": "1.49w", + "tag": "近地铁6号线巨峰路站" + }, + { + "id": 58709, + "name": "新光大楼", + "area": "浦东", + "img": "https://img.mapmapping.com/f8e1dffb-fa85-457b-8a97-fb019239bdf6.png?imageView2/2/w/240/h/200/q/75", + "town": "潍坊", + "schoolTag": "浦东新区明珠小学A区", + "sellerNum": "3", + "leaseNum": "2", + "price": "11.95w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.23234474870607700", + "lng": "121.52462643697852000", + "tags": [ + "近地铁9号线商城路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.49w", + "tag": "近地铁9号线商城路站" + }, + { + "id": 59322, + "name": "盛世年华", + "area": "浦东", + "img": "https://img.mapmapping.com/83727155-29f5-47d9-b54d-4042d9e625ec.png?imageView2/2/w/240/h/200/q/75", + "town": "源深", + "schoolTag": "浦东新区福山花园外国语小学", + "sellerNum": "4", + "leaseNum": "11", + "price": "14.94w", + "formatPrice": null, + "year": "2005年建成", + "lat": "31.23494392135101000", + "lng": "121.54545194344688000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "2005年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58326, + "name": "梅园七街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/30cf630f-830b-4e43-8dc5-648bb7dfb760.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "2", + "leaseNum": "2", + "price": "11.13w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24378516838488700", + "lng": "121.53455423998314000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58599, + "name": "陈家门小区", + "area": "浦东", + "img": "https://img.mapmapping.com/7a9f43ef-67c4-4168-bf24-0190689d331d.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区梅园小学", + "sellerNum": "3", + "leaseNum": "1", + "price": "9.27w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24726096884606800", + "lng": "121.53135690118148000", + "tags": [ + "近地铁4号线浦东大道站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁4号线浦东大道站" + } + ], + "pageNum": 1, + "pageSize": 10, + "size": 10, + "startRow": 1, + "endRow": 10, + "pages": 474, + "prePage": 0, + "nextPage": 2, + "isFirstPage": true, + "isLastPage": false, + "hasPreviousPage": false, + "hasNextPage": true, + "navigatePages": 8, + "navigatepageNums": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ], + "navigateFirstPage": 1, + "navigateLastPage": 8 + }, + "timestamp": 1630575199849 +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json new file mode 100644 index 000000000..125089046 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json @@ -0,0 +1,65 @@ +{ + "code": "000", + "desc": "操作成功", + "content": [ + { + "id": 272113, + "name": "第六师范学校第二附属小学西校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "小学", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "41", + "sellNum": "188", + "plotPrice": "8.69", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + }, + { + "id": 273287, + "name": "进才中学北校苗圃路校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "初中", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "85", + "sellNum": "339", + "plotPrice": "9.33", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + } + ], + "timestamp": 1630576702404 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java new file mode 100644 index 000000000..b02953be7 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java @@ -0,0 +1,79 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseDetail { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "imgsUrl") + private String imgsUrl; + @JSONField(name = "area") + private String area; + @JSONField(name = "town") + private String town; + @JSONField(name = "price") + private String price; + @JSONField(name = "priceDesc") + private String priceDesc; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "dealNum") + private String dealNum; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "houseNum") + private String houseNum; + @JSONField(name = "building") + private String building; + @JSONField(name = "propertyFee") + private String propertyFee; + @JSONField(name = "propertyCompany") + private String propertyCompany; + @JSONField(name = "developers") + private String developers; + @JSONField(name = "lat") + private Double lat; + @JSONField(name = "lng") + private Double lng; + @JSONField(name = "imgArr") + private List imgArr; + @JSONField(name = "focus") + private Boolean focus; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "view") + private Integer view; + @JSONField(name = "address") + private String address; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java new file mode 100644 index 000000000..cd49ae026 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java @@ -0,0 +1,107 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseList { + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "total") + private Integer total; + @JSONField(name = "list") + private List list; + @JSONField(name = "pageNum") + private Integer pageNum; + @JSONField(name = "pageSize") + private Integer pageSize; + @JSONField(name = "size") + private Integer size; + @JSONField(name = "startRow") + private Integer startRow; + @JSONField(name = "endRow") + private Integer endRow; + @JSONField(name = "pages") + private Integer pages; + @JSONField(name = "prePage") + private Integer prePage; + @JSONField(name = "nextPage") + private Integer nextPage; + @JSONField(name = "isFirstPage") + private Boolean isFirstPage; + @JSONField(name = "isLastPage") + private Boolean isLastPage; + @JSONField(name = "hasPreviousPage") + private Boolean hasPreviousPage; + @JSONField(name = "hasNextPage") + private Boolean hasNextPage; + @JSONField(name = "navigatePages") + private Integer navigatePages; + @JSONField(name = "navigatepageNums") + private List navigatepageNums; + @JSONField(name = "navigateFirstPage") + private Integer navigateFirstPage; + @JSONField(name = "navigateLastPage") + private Integer navigateLastPage; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "area") + private String area; + @JSONField(name = "img") + private String img; + @JSONField(name = "town") + private String town; + @JSONField(name = "schoolTag") + private String schoolTag; + @JSONField(name = "sellerNum") + private String sellerNum; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "price") + private String price; + @JSONField(name = "formatPrice") + private Object formatPrice; + @JSONField(name = "year") + private String year; + @JSONField(name = "lat") + private String lat; + @JSONField(name = "lng") + private String lng; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "view") + private String view; + @JSONField(name = "tag") + private String tag; + } + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java new file mode 100644 index 000000000..1beb5153b --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -0,0 +1,85 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class School { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private List content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "nature") + private Object nature; + @JSONField(name = "city") + private String city; + @JSONField(name = "area") + private String area; + @JSONField(name = "type") + private String type; + @JSONField(name = "quality") + private String quality; + @JSONField(name = "location") + private Object location; + @JSONField(name = "picture") + private String picture; + @JSONField(name = "phone") + private String phone; + @JSONField(name = "distanceNum") + private Object distanceNum; + @JSONField(name = "plotNum") + private String plotNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "plotPrice") + private String plotPrice; + @JSONField(name = "address") + private Object address; + @JSONField(name = "lat") + private Object lat; + @JSONField(name = "lng") + private Object lng; + @JSONField(name = "scope") + private Object scope; + @JSONField(name = "score") + private Object score; + @JSONField(name = "scopeArr") + private Object scopeArr; + @JSONField(name = "scopeTag") + private Object scopeTag; + @JSONField(name = "tag") + private Object tag; + @JSONField(name = "shortName") + private Object shortName; + @JSONField(name = "view") + private Object view; + @JSONField(name = "construct") + private Object construct; + @JSONField(name = "echelon") + private Object echelon; + @JSONField(name = "town") + private Object town; + } +} From 729b2a246c78f6e3580111eec2fac8cbd5c6418a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 3 Sep 2021 09:20:48 +0800 Subject: [PATCH 097/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=80=E6=89=8B?= =?UTF-8?q?=E6=88=BF=E6=88=90=E4=BA=A4=E4=BF=A1=E6=81=AF=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/NewHouseFangDi.groovy | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy new file mode 100644 index 000000000..b1b702fb6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy @@ -0,0 +1,70 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +/** + * @author shenjian* @since 2021/9/2 + */ +class NewHouseFangDi { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.fangdi.com.cn") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36") + headers.put("Content-Type", "application/x-www-form-urlencoded;charset=utf-8") + headers.put("Accept", "application/json, text/javascript, */*; q=0.01") + headers.put("X-Requested-With", "XMLHttpRequest") + headers.put("Origin", "http://www.fangdi.com.cn") + headers.put("Referer", "http://www.fangdi.com.cn/new_house/new_house_list_fast.html?dicPriceID=&dicRegionID=&flatID=&districtID=5391be55e75e5506&check_input2=105C") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Accept-Encoding", "gzip, deflate") + headers.put("Accept-Language", "zh-CN,zh;q=0.9") + headers.put("Cookie", "www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; www.fangdi.com.cn=www.fangdi.com.cn_rs3; FSSBBIl1UgzbN7N80S=QS7tHXu4vqldPlDeqJkUCGoyqBmqHzTtyqvrk2ibcRKGVcbPOJ7ERuh5qYgHVD4l; www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; FSSBBIl1UgzbN7Nenable=true; JSESSIONID1=jw-mUJZsTiFYkXShzn6asMa401YoEZs-0WzhNHrUGId4Jh1_14tj!-338668155; FSSBBIl1UgzbN7N80T=4t2lc885DGECHVNGMwL4w1LcI9v43OwswK7ZOXM5ZbcyVF1UvDPUs59KVOQkq_2hxc0iJrZi3gBiildN1Xfh7xNz81qFz12KwlUJrncPg8mejTeav9tTOgiiPXMZZfJ4rltOCYgOzLLqbj5nG9esEse9UWPYnFdkTWvjpJUTfdyIGg9vB_S9ABAvmir3e89JTbzvfK4.i_IRPFED2hsteZcTiIpNWteOxZMMeDPSnKJei2_Hqhmn7LQMVZanibREAJ8Ah7IxIAwcMtRqAtZLLRpcQMtwnqkEoUPHf435yJVKL20fHui11nhVV4MxTHo9pcoz6L.yhuezCyP5NGTdpV14XV_yBCBhNkSXKWbO.lWAEUQ8yBkMwLLgaYY34sg0Bz_0") + + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + try { + Map params = new HashMap(); +// params.put("districtID", "5391be55e75e5506") +// params.put("dicRegionID", "") +// params.put("dicPriceID", "") +// params.put("flatID", "") + params.put("districtID", "5391be55e75e5506") + params.put("dicRegionID", "") + params.put("stateID", "4d29bb4241910ce3") + params.put("houseAreaID", "0") + params.put("dicAvgpriceID", "0") + params.put("dicPositionID", "") + params.put("houseTypeID", "0") + params.put("houseTypeID", "0") + params.put("address", "") + params.put("openingID", "") + params.put("projectName", "") + params.put("currentPage", "1") + str = re.post("http://www.fangdi.com.cn/service/freshHouse/getFastHosueList.action", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + def data = JSONObject.parseObject(str).get("data") + print(data) + } +} From d78e6a664a840cc0fa34536d01f6640243fd58d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 3 Sep 2021 14:04:54 +0800 Subject: [PATCH 098/118] =?UTF-8?q?=E5=95=86=E5=9C=88=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E6=96=B0=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/BusinessLatLng.groovy | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy new file mode 100644 index 000000000..519e5de21 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 商圈经纬度 + */ +class BusinessLatLng { + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(1) + + static void main(String[] args) { + dealPoi(new BasicDBObject("city", "上海").append("region", "浦东新区")) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + collection.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def address = city + obj.region + obj.address; + def location = GaoDeUtils.parserByGaode(city, address); + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度"))) + } else { + collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} From 776b32e1aaa2b576f891bb79f28cc353a4550d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 3 Sep 2021 14:22:59 +0800 Subject: [PATCH 099/118] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AF=B9=E5=8F=A3?= =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/SeaweedSchoolLatLng.groovy | 86 +++++++++++++++++++ .../shenjian/seaweed/SeaweedSchool.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 49 ++++++++++- 3 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy new file mode 100644 index 000000000..d511bb745 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 对口小学初中经纬度 + */ +class SeaweedSchoolLatLng { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + static ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0); + collection.find(new BasicDBObject()).batchSize(1000).each {objT -> + final DBObject obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def name = dbObject.name + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + if (dbObject.lng != null) { + String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) + dbObject.address = address + } + } + } + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def name = dbObject.name + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + if (dbObject.lng != null) { + String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) + dbObject.address = address + } + } + } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + atomicLong.incrementAndGet() + synchronized (this) { + if (atomicLong.longValue() % 100 == 0) { + println(atomicLong.longValue()) + } + } + } + } + }) + }; + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index 5981472a2..73545a2d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -19,7 +19,9 @@ import org.bson.Document import org.jdom.Content; /** - * @author shenjian* @since 2021/9/2 + * 对口小学、初中 + * @author shenjian + * @since 2021/9/2 */ class SeaweedSchool { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 1c0188047..5334643d3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil import com.fangjia.core.util.HttpRequest +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil @@ -214,8 +215,6 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 System.exit(1) - } else { - break } } return result @@ -228,6 +227,52 @@ class GaoDeUtils { return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) } + /** + * 经纬度转地址
+ * 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位 + * @author yinjihuan + * @date 2016年9月9日 + * @param location + * @return + */ + static def getGaoDeAddress(location) { + Map params = new HashMap() + params.put("key", GAO_DE_AK[keyUsed.intValue()]) + params.put("location", location) + String content + try { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) + Map infoMap = DataHandleUtils.JsonToMap(json); + if (infoMap && infoMap.info == "OK") { + def regeocode = infoMap.regeocode + if (regeocode) { + return regeocode.formatted_address; + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return null + } + /** * 获取地址经纬度 */ From f267f393fe99315173cb8077a9e77d8b1ddbdec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 5 Sep 2021 11:02:26 +0800 Subject: [PATCH 100/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AF=B9=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 6 +- .../shenjian/seaweed/SeaweedSchool.groovy | 91 +++++++++++----- .../script/statistics/seaweed/School.java | 101 +++++++----------- 3 files changed, 109 insertions(+), 89 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 01d0e126d..517fcf4ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,11 +1,15 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - + // 121.431870,31.208610 + List list = GaoDeUtils.searchPoi("上海", "浦东新区新场实验小学新城校区") + print(list) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index 73545a2d8..b988b6a9d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -44,10 +44,10 @@ class SeaweedSchool { Map headers = headerMap("") HttpRequest re = new HttpRequest(); String str = null - final String area = "浦东新区"; + final String area = "浦东"; final int perPage = 10 final int total = 4733; - for (int page = 15; page < total / perPage; page++) { + for (int page = 1; page < total / perPage; page++) { Thread.sleep(1000) print("当前页:" + page) try { @@ -71,16 +71,18 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(1000) + Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) + // 对口幼儿园 + School nurserySchool = getNurserySchool(re, id, headers) //# 对口小学 School primarySchool = getPrimarySchool(re, id, headers) //# 对口中学 School middleSchool = getMiddleSchool(re, id, headers) String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) - saveData(houseDetail, primarySchool, middleSchool, url); + saveData(houseDetail, nurserySchool, primarySchool, middleSchool, url); } } } @@ -88,7 +90,7 @@ class SeaweedSchool { } } - private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) { + private static void saveData(HouseDetail houseDetail, School nurserySchool, School primarySchool, School middleSchool, def url) { DBObject dbObject = new BasicDBObject() dbObject.put("url", url) dbObject.put("root", "沪上学区房") @@ -97,38 +99,60 @@ class SeaweedSchool { dbObject.put("region", houseDetail.getContent().getArea()) dbObject.put("name", houseDetail.getContent().getName()) dbObject.put("address", houseDetail.getContent().getAddress()) - List primarySchools = primarySchool.getContent(); + dbObject.put("lng", houseDetail.getContent().getLng()) + dbObject.put("lat", houseDetail.getContent().getLat()) + + List nurserySchools = nurserySchool.getContent().getList(); + if (CollectionUtils.isNotEmpty(nurserySchools)) { + List schools = buildSchool(nurserySchools) + dbObject.put("nurserySchool", schools) + } + + List primarySchools = primarySchool.getContent().getList();; if (CollectionUtils.isNotEmpty(primarySchools)) { - List schools = new ArrayList<>() - for (School.ContentDTO contentDTO : primarySchools) { - DBObject schoolObject = new BasicDBObject(); - if ("小学".equals(contentDTO.getType())) { - schoolObject.put("name", contentDTO.getName()) - schoolObject.put("quality", contentDTO.getQuality()) - schools.add(schoolObject) - } - } + List schools = buildSchool(primarySchools) dbObject.put("primarySchool", schools) } - List middleSchools = middleSchool.getContent(); + List middleSchools = middleSchool.getContent().getList();; if (CollectionUtils.isNotEmpty(middleSchools)) { - List schools = new ArrayList<>() - for (School.ContentDTO contentDTO : middleSchools) { - DBObject schoolObject = new BasicDBObject(); - if ("初中".equals(contentDTO.getType())) { - schoolObject.put("name", contentDTO.getName()) - schoolObject.put("quality", contentDTO.getQuality()) - schools.add(schoolObject) - } - } + List schools = buildSchool(middleSchools) dbObject.put("middleSchool", schools) } - collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) } + private static List buildSchool(List currSchools) { + List schools = new ArrayList<>() + for (School.ContentDTO.ListDTO contentDTO : currSchools) { + DBObject schoolObject = new BasicDBObject(); + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(contentDTO.getTags())) { + if (contentDTO.getTags().size() >= 3) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", contentDTO.getTags().get(2)) + } else if (contentDTO.getTags().size() == 2) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", '') + } else { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", '') + schoolObject.put("batch", '') + } + } + schoolObject.put("area", contentDTO.getArea()) + schoolObject.put("town", contentDTO.getTown()) + schoolObject.put("distance", contentDTO.getDistance()) + schoolObject.put("address", contentDTO.getAddress()) + schools.add(schoolObject) + } + return schools + } + + + private static HouseDetail getHouseDetail(def re, def id, def headers) { Map params = new HashMap(); params.put("id", String.valueOf(id)) @@ -137,6 +161,17 @@ class SeaweedSchool { return JSONObject.parseObject(str, HouseDetail.class) } + private static School getNurserySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "1") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + private static School getPrimarySchool(def re, def id, def headers) { Map params = new HashMap(); params.put("id", String.valueOf(id)) @@ -144,7 +179,7 @@ class SeaweedSchool { params.put("page", "1") params.put("perPage", "2") params.put("source", "1") - def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); return JSONObject.parseObject(str, School.class) } @@ -155,7 +190,7 @@ class SeaweedSchool { params.put("page", "1") params.put("perPage", "4") params.put("source", "1") - def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); return JSONObject.parseObject(str, School.class) } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java index 1beb5153b..992727e92 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -1,6 +1,6 @@ package com.fangjia.script.statistics.seaweed; -import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.NoArgsConstructor; @@ -15,71 +15,52 @@ import java.util.List; public class School { - @JSONField(name = "code") + @JsonProperty("code") private String code; - @JSONField(name = "desc") + @JsonProperty("desc") private String desc; - @JSONField(name = "content") - private List content; - @JSONField(name = "timestamp") + @JsonProperty("content") + private ContentDTO content; + @JsonProperty("timestamp") private Long timestamp; @NoArgsConstructor @Data public static class ContentDTO { - @JSONField(name = "id") - private Integer id; - @JSONField(name = "name") - private String name; - @JSONField(name = "nature") - private Object nature; - @JSONField(name = "city") - private String city; - @JSONField(name = "area") - private String area; - @JSONField(name = "type") - private String type; - @JSONField(name = "quality") - private String quality; - @JSONField(name = "location") - private Object location; - @JSONField(name = "picture") - private String picture; - @JSONField(name = "phone") - private String phone; - @JSONField(name = "distanceNum") - private Object distanceNum; - @JSONField(name = "plotNum") - private String plotNum; - @JSONField(name = "sellNum") - private String sellNum; - @JSONField(name = "plotPrice") - private String plotPrice; - @JSONField(name = "address") - private Object address; - @JSONField(name = "lat") - private Object lat; - @JSONField(name = "lng") - private Object lng; - @JSONField(name = "scope") - private Object scope; - @JSONField(name = "score") - private Object score; - @JSONField(name = "scopeArr") - private Object scopeArr; - @JSONField(name = "scopeTag") - private Object scopeTag; - @JSONField(name = "tag") - private Object tag; - @JSONField(name = "shortName") - private Object shortName; - @JSONField(name = "view") - private Object view; - @JSONField(name = "construct") - private Object construct; - @JSONField(name = "echelon") - private Object echelon; - @JSONField(name = "town") - private Object town; + @JsonProperty("total") + private Integer total; + @JsonProperty("hasNextPage") + private Boolean hasNextPage; + @JsonProperty("list") + private List list; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JsonProperty("id") + private Integer id; + @JsonProperty("name") + private String name; + @JsonProperty("area") + private String area; + @JsonProperty("shortName") + private Object shortName; + @JsonProperty("picture") + private String picture; + @JsonProperty("distance") + private String distance; + @JsonProperty("plotPrice") + private String plotPrice; + @JsonProperty("topInfo") + private Object topInfo; + @JsonProperty("view") + private String view; + @JsonProperty("town") + private String town; + @JsonProperty("address") + private String address; + @JsonProperty("tags") + private List tags; + } } } From d961902f86c17f7c8e17e2d091b7a7fd7726f378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 5 Sep 2021 13:52:50 +0800 Subject: [PATCH 101/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=90=8D=E5=AD=97?= =?UTF-8?q?=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/SeaweedSchoolLatLng.groovy | 62 +++++++++---------- .../shenjian/seaweed/SeaweedSchool.groovy | 3 +- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy index d511bb745..93bf399b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -7,7 +7,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong @@ -19,7 +18,7 @@ class SeaweedSchoolLatLng { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) - static ExecutorService executorService = Executors.newFixedThreadPool(20) + static ExecutorService executorService = Executors.newFixedThreadPool(1) static void main(String[] args) { AtomicLong atomicLong = new AtomicLong(0); @@ -29,45 +28,40 @@ class SeaweedSchoolLatLng { @Override void run() { try { - BasicDBList primarySchools = obj.primarySchool - BasicDBList middleSchools = obj.middleSchool - if (primarySchools != null) { - for (BasicDBObject dbObject : primarySchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def address = obj.address + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } } - } - if (middleSchools != null) { - for (BasicDBObject dbObject : middleSchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } - collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } catch(Exception ex) { ex.printStackTrace() } finally { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index b988b6a9d..b3ab13fef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -48,7 +48,6 @@ class SeaweedSchool { final int perPage = 10 final int total = 4733; for (int page = 1; page < total / perPage; page++) { - Thread.sleep(1000) print("当前页:" + page) try { Map params = new HashMap(); @@ -71,7 +70,6 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) @@ -146,6 +144,7 @@ class SeaweedSchool { schoolObject.put("town", contentDTO.getTown()) schoolObject.put("distance", contentDTO.getDistance()) schoolObject.put("address", contentDTO.getAddress()) + schoolObject.put("name", contentDTO.getName()) schools.add(schoolObject) } return schools From 8d4c94431ce28ae8cabe56f72305a56ffbdcd866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 5 Sep 2021 13:52:50 +0800 Subject: [PATCH 102/118] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=90=8D=E5=AD=97?= =?UTF-8?q?=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/SeaweedSchoolLatLng.groovy | 163 ++++++++++++++---- .../shenjian/seaweed/SeaweedSchool.groovy | 3 +- 2 files changed, 130 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy index d511bb745..ad1676b7d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -7,7 +7,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong @@ -18,10 +17,17 @@ import java.util.concurrent.atomic.AtomicLong class SeaweedSchoolLatLng { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + static DBCollection school_collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) static ExecutorService executorService = Executors.newFixedThreadPool(20) static void main(String[] args) { + //latLng() + updateAttribute() + executorService.shutdown() + } + + static void updateAttribute() { AtomicLong atomicLong = new AtomicLong(0); collection.find(new BasicDBObject()).batchSize(1000).each {objT -> final DBObject obj = objT @@ -29,45 +35,68 @@ class SeaweedSchoolLatLng { @Override void run() { try { - BasicDBList primarySchools = obj.primarySchool - BasicDBList middleSchools = obj.middleSchool - if (primarySchools != null) { - for (BasicDBObject dbObject : primarySchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def address = obj.address + boolean update = false + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'district_name': obj.name)) + def quality = '' + + if (primarySchools != null) { + if (schoolObj && schoolObj.attribute) { + quality = schoolObj.attribute + } + for (BasicDBObject dbObject : primarySchools) { + if (dbObject.name == '第六师范学校第二附属小学西校区' || String.valueOf(dbObject.name).startsWith('第六师范附属小学') + || dbObject.name == '上海市第六师范第二附属小学七号桥校区' || dbObject.name == '康城学校康佳校区' || dbObject.name == '三墩学校') { + dbObject.quality = '普通' + update = true + } else if (dbObject.name == '上海市实验学校东校(小学)' || dbObject.name == '浦东新区竹园小学沪东校区') { + dbObject.quality = '市重点' + update = true + } else if (dbObject.name == '浦东新区上南二村小学南校区' || dbObject.name == '浦东新区新时代小学大华校区' + || dbObject.name == '浦东新区明珠小学A区') { + dbObject.quality = '区重点' + update = true + } + else if (quality != '' && schoolObj.category == '小学') { + dbObject.quality = quality + update = true + } else { + update = updateDbObjectQuality(dbObject); } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } } - } - if (middleSchools != null) { - for (BasicDBObject dbObject : middleSchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def update2 = false + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + if (dbObject.name == '华东师范大学张江实验中学' || dbObject.name == '10上海市实验学校(初中)(全市招生)' || dbObject.name == '上海市实验学校附属光明学校总校区' + || dbObject.name == '上海市浦东新区进才实验中学南校' || String.valueOf(dbObject.name).startsWith('建平中学') || String.valueOf(dbObject.name).startsWith('建平实验中学') + || dbObject.name == '张江集团中学' || dbObject.name == '洋泾-菊园实验学校' || dbObject.name == '浦东外国语学校东校' || dbObject.name == '浦东新区建平中学南校' || String.valueOf(dbObject.name).startsWith('育人中学') + || dbObject.name == '致远中学' || String.valueOf(dbObject.name).startsWith('进才中学') || String.valueOf(dbObject.name).startsWith('进才实验中学')) { + dbObject.quality = '市重点' + update2 = true + } else if (dbObject.name == '华东师范大学附属东昌中学南校潍坊校区' || dbObject.name == '下沙学校鹤沙路校区' || dbObject.name == '傅雷中学关岳路校区' + || dbObject.name == '傅雷中学瑞和路校区' || dbObject.name == '华东师范大学附属东昌中学南校张江校区' || dbObject.name == '南汇第一中学卫星东路校区' + || dbObject.name == '康城学校(拯安校区)(初中)' || dbObject.name == '建平香梅中学' || dbObject.name == '沪新中学' || dbObject.name == '浦东模范中学' + || dbObject.name == '浦东模范中学东校' || dbObject.name == '浦东模范实验中学' || String.valueOf(dbObject.name).startsWith('澧溪中学') || + dbObject.name == '清流中学' || String.valueOf(dbObject.name).startsWith('王港中学') || dbObject.name == '罗山中学' + || dbObject.name == '金杨中学' || dbObject.name == '香山中学' || dbObject.name == '高行中学') { + dbObject.quality = '区重点' + update2 = true + } else if (quality != '' && schoolObj.category == '初中') { + dbObject.quality = quality + update2 = true + } else { + update2 = updateDbObjectQuality(dbObject) } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } + } + if (update || update2) { + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } } - collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } catch(Exception ex) { ex.printStackTrace() } finally { @@ -81,6 +110,72 @@ class SeaweedSchoolLatLng { } }) }; - executorService.shutdown() + } + static boolean updateDbObjectQuality(BasicDBObject dbObject) { + if (dbObject.quality == '公办') { + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'school_name': dbObject.name)) + if (schoolObj && schoolObj.attribute) { + dbObject.quality = schoolObj.attribute + return true + } + } + return false + } + + static void latLng() { + AtomicLong atomicLong = new AtomicLong(0); + collection.find(new BasicDBObject()).batchSize(1000).each {objT -> + final DBObject obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = obj.address + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + atomicLong.incrementAndGet() + synchronized (this) { + if (atomicLong.longValue() % 100 == 0) { + println(atomicLong.longValue()) + } + } + } + } + }) + }; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index b988b6a9d..b3ab13fef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -48,7 +48,6 @@ class SeaweedSchool { final int perPage = 10 final int total = 4733; for (int page = 1; page < total / perPage; page++) { - Thread.sleep(1000) print("当前页:" + page) try { Map params = new HashMap(); @@ -71,7 +70,6 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) @@ -146,6 +144,7 @@ class SeaweedSchool { schoolObject.put("town", contentDTO.getTown()) schoolObject.put("distance", contentDTO.getDistance()) schoolObject.put("address", contentDTO.getAddress()) + schoolObject.put("name", contentDTO.getName()) schools.add(schoolObject) } return schools From 35ab32f65ebbb9b073035fbf0772e5d316be3e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 9 Sep 2021 15:44:24 +0800 Subject: [PATCH 103/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/dingding/DingTalkUtils.groovy | 2 +- .../temp/shenjian/poi/CommitteeLatLng.groovy | 17 +++++++++++------ .../script/temp/shenjian/util/GaoDeUtils.groovy | 12 +++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 209ab9fe2..795b1556a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,7 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd") ; + Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index 1f5f696cf..71d8c3bec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.poi +import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -18,7 +19,7 @@ import java.util.concurrent.Executors class CommitteeLatLng { static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) - static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService executorService = Executors.newFixedThreadPool(1) static void main(String[] args) { dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) @@ -37,12 +38,16 @@ class CommitteeLatLng { city = obj.province_name } def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; - def location = GaoDeUtils.parserByGaode(city, address); - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) + // 不准确 + // def location = GaoDeUtils.parserByGaode(city, address); + List amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "130106|130105|130104") + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) - .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度"))) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) } else { villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 5334643d3..8496f3f01 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -128,9 +128,9 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); if (json == null) { - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() @@ -169,11 +169,13 @@ class GaoDeUtils { /** * 查找关键字经纬度 */ - static List searchPoi(String city, String keywords){ + static List searchPoi(String city, String keywords, String region, String typecodes){ def params = [:] params.key = GAO_DE_AK[keyUsed.intValue()] params.keywords = keywords params.city = city + params.region = region + params.types = typecodes params.put("extensions", "all") params.put("citylimit","true") params.put("offset","20") @@ -184,9 +186,9 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); if (json == null) { - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() From 613cd0db9b868bfb3f8473ebafad50d83fa1c620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 9 Sep 2021 16:16:38 +0800 Subject: [PATCH 104/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy | 6 ++++++ .../com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index 71d8c3bec..7edea330f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.poi +import com.fangjia.core.util.CosineSimilarAlgorithm import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.MongoUtils @@ -43,6 +44,11 @@ class CommitteeLatLng { List amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "130106|130105|130104") if (amapPoi && amapPoi.size() > 0) { def poi = amapPoi[0] + double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, obj.name) + if (similarity < 0.7) { + amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "") + } + poi = amapPoi[0] String[] locations = poi.location.split(",") def lng = Double.valueOf(locations[0]) def lat = Double.valueOf(locations[1]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 8496f3f01..c75696cd6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import org.apache.commons.lang.StringUtils import java.util.concurrent.atomic.AtomicInteger @@ -175,7 +176,9 @@ class GaoDeUtils { params.keywords = keywords params.city = city params.region = region - params.types = typecodes + if (StringUtils.isNotBlank(typecodes)) { + params.types = typecodes + } params.put("extensions", "all") params.put("citylimit","true") params.put("offset","20") From 69cb427f83fd7a6799ef2b49a04584b8f024a51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 10 Sep 2021 11:42:36 +0800 Subject: [PATCH 105/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/CommitteeLatLng.groovy | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index 7edea330f..9a7ff1bb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -20,7 +20,7 @@ import java.util.concurrent.Executors class CommitteeLatLng { static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) - static ExecutorService executorService = Executors.newFixedThreadPool(1) + static ExecutorService executorService = Executors.newFixedThreadPool(20) static void main(String[] args) { dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) @@ -37,10 +37,10 @@ class CommitteeLatLng { def city = obj.city_name if (city == "市辖区") { city = obj.province_name + } else if (city == "省直辖县级行政区划") { + city = obj.area_name } def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; - // 不准确 - // def location = GaoDeUtils.parserByGaode(city, address); List amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "130106|130105|130104") if (amapPoi && amapPoi.size() > 0) { def poi = amapPoi[0] @@ -55,7 +55,18 @@ class CommitteeLatLng { villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) } else { - villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "") + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) + + } else { + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } } println("已处理小区:" + obj) } From d20b50d7617c939dd6cf8e310ee42dc9dc45d2a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 10 Sep 2021 14:45:14 +0800 Subject: [PATCH 106/118] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=92=89=E9=92=89?= =?UTF-8?q?=E5=AF=BC=E5=87=BA=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/fangjia/script/statistics/ApiUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 7c87580de..746556da5 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -347,7 +347,7 @@ public class ApiUtil { throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:getTaskDetail"); } } catch (Exception e) { - e.printStackTrace(); + //e.printStackTrace(); } return detail; } From 6b06ee211e4b1ba78cd51111123d5e9b272d2831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 13 Sep 2021 09:22:23 +0800 Subject: [PATCH 107/118] =?UTF-8?q?excel=E4=B8=8D=E5=90=8C=E7=89=88?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 2e06fc2a1..3c1644892 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -366,12 +366,14 @@ org.apache.poi poi - 3.17 + + 3.10-FINAL org.apache.poi poi-ooxml - 3.17 + + 3.10-FINAL xml-apis @@ -476,21 +478,18 @@ aliyun-java-sdk-address-purification 1.0.0 + - com.alibaba - easyexcel - 2.2.8 - - - org.projectlombok - lombok - 1.18.20 - provided - - - org.msgpack - msgpack - 0.6.12 + com.eclipsesource.j2v8 + j2v8_win32_x86_64 + 4.6.0 + + + + elastic.co + https://artifacts.elastic.co/maven + + From f9a60cfa954b09fbf70a3c52db77895a02404d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 14 Sep 2021 14:46:14 +0800 Subject: [PATCH 108/118] =?UTF-8?q?=E9=93=B6=E8=81=94=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 59 ++++++++-------- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 34 ++++----- .../poi/YinlianLngLatAndAround.groovy | 69 +++++++++++++++++++ .../temp/shenjian/util/ExcelUtils.groovy | 9 +++ .../temp/shenjian/util/GaoDeUtils.groovy | 36 ++++++++++ 5 files changed, 163 insertions(+), 44 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 3c1644892..bcebb2b2b 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -1,5 +1,5 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 com.fangjia fangjia-script @@ -62,7 +62,7 @@ - org.eclipse.m2e @@ -141,11 +141,11 @@ activemq-pool 5.5.0
- - cn.emay - emayclient - 4.3 - + + cn.emay + emayclient + 4.3 + org.apache.activemq activemq-core @@ -366,20 +366,20 @@ org.apache.poi poi - - 3.10-FINAL + 3.17 + org.apache.poi poi-ooxml - - 3.10-FINAL - - - xml-apis - xml-apis - - + 3.17 + + + + xml-apis + xml-apis + +