From ddf08d464dabc5c81dea15ca20e5907307a58aa2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Apr 2021 12:34:04 +0800 Subject: [PATCH 001/212] =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E7=BA=A7=E8=81=94?= =?UTF-8?q?=E5=9F=8E=E5=B8=82=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/DealPriceDetailExport.groovy | 183 ++++++++++++++++++ .../script/temp/shenjian/PPE15Export.groovy | 158 +++++++++++++++ .../temp/shenjian/PPE15ExportTest.groovy | 130 +++++++++++++ .../script/temp/shenjian/RentExport.groovy | 153 +++++++++++++++ .../temp/shenjian/TotalTestExport.groovy | 92 +++++++++ .../shenjian/ValueRateCityInfoExport.groovy | 91 +++++++++ .../shenjian/es/CopyDataForAnnPricePre.groovy | 124 ++++++++++++ .../shenjian/es/SyncDistrictPriceStat.groovy | 98 ++++++++++ .../shenjian/es/SyncDistrictPriceStat2.groovy | 78 ++++++++ .../es/util/RestHighLevelClientUtils.groovy | 35 ++++ 10 files changed, 1142 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy new file mode 100644 index 000000000..9f532247a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/DealPriceDetailExport.groovy @@ -0,0 +1,183 @@ +package com.fangjia.script.temp.shenjian + +import com.alibaba.excel.EasyExcel +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject + +import java.util.concurrent.* + +/** + * 成交详情导出 + * + * @author shenjian* @since 2021/3/29 + */ +class DealPriceDetailExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43", 28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43", 28017) + static final Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(10) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海", "广州", "深圳", "重庆", "郑州", "长沙", "西安", "成都", "东莞", "佛山", "杭州", + "合肥", "南京", "青岛", "沈阳", "苏州", "天津", "武汉", "珠海", "中山", "长春", "扬州", "烟台", + "徐州", "无锡", "温州", "太原", "台州", "石家庄", "绍兴", "厦门", "泉州", "宁波", "南通", "南宁", + "南昌", "兰州", "常州", "大连", "福州", "贵阳", "哈尔滨", "海口", "惠州", "济南", "嘉兴", "金华", + "昆明", "遵义", "淄博", "株洲", "镇江", "肇庆", "漳州", "湛江", "岳阳", "银川", "宜昌", "盐城", + "芜湖", "乌鲁木齐", "唐山", "泰州", "汕头", "三亚", "清远", "秦皇岛", "马鞍山", "洛阳", "柳州", + "连云港", "廊坊", "九江", "吉林", "淮安", "湖州", "呼和浩特", "沧州", "保定", "安庆", "张家口", + "西宁", "日照", "拉萨", "锦州", "大同", "常德", "北海", "文昌", "陵水", "保亭"] +// def cities = ["常州"] + + def headArr = ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量", "小区均价", "比率"] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + + CountDownLatch latch = new CountDownLatch(cities.size()) + DBObject filter = new BasicDBObject() + filter.put("city", 1) + filter.put("region", 1) + filter.put("district_name", 1) + filter.put("total_price", 1) + filter.put("trade_date", 1) + filter.put("avg_price", 1) + filter.put("area", 1) + filter.put("direction", 1) + filter.put("fitment", 1) + filter.put("height", 1) + filter.put("floor", 1) + filter.put("unit_num", 1) + filter.put("house_num", 1) + filter.put("room_num", 1) + filter.put("hall", 1) + filter.put("room", 1) + filter.put("toilet", 1) + for (def city : cities) { + final currCity = city + executorService.execute(new Runnable() { + @Override + void run() { + try { + final def ret = new CopyOnWriteArrayList() + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", currCity).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + //ret << ["城市", "区域", "小区名", "总价", "均价(每平米)", "成交日期", "面积", "朝向", "装修情况", "总楼层", "所属楼层", "单元号", "楼号", "室号", "厅数量", "室数量", "卫数量"] + DBObject query = new BasicDBObject(); + query.put("city", currCity) + query.put("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START))) + + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", currCity).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(query, filter).batchSize(10000).each { tempObj -> + final obj = tempObj + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def row = [] + row[0] = currCity + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.total_price + row[4] = obj.avg_price + if (obj.trade_date) { + row[5] = DateUtils.formatDateByPattern(obj.trade_date, "yyyy-MM-dd") + } + row[6] = obj.area + row[7] = obj.direction + row[8] = obj.fitment + row[9] = obj.height + row[10] = obj.floor + row[11] = obj.unit_num + row[12] = obj.house_num + row[13] = obj.room_num + row[14] = obj.hall + row[15] = obj.room + row[16] = obj.toilet + def objOur = map.get(obj.region + obj.district_name) + calcOurAvgPrice(currCity, obj.trade_date, obj.avg_price, objOur, row) + ret.add(row) + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + println("出现异常111111.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + } + districtLatch.await() + EasyExcel.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx").head(head).sheet("成交信息").doWrite(ret); + //ExcelUtils.write("F:\\数据\\成交详情\\" + currCity + "小区成交详情.xlsx", ret, "成交信息") + println("当前处理城市" + currCity + " " + ret.size()) + } catch (Exception ex) { + println("出现异常22222.。。。。。。。。。。。") + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + executorService.shutdown() + districtExecutorService.shutdown() + } + + private static calcOurAvgPrice(def city, def tradeDate, def avgPrice, def objOur, def row) { + // 最相近一周的该小区均价 + Date nextWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = org.apache.commons.lang3.time.DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", objOur.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + objOur.region + " " + objOur.name) + row[17] = "" + row[18] = "" + } else { + row[17] = weekendAvgPrice + row[18] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(weekendAvgPrice) / Double.valueOf(avgPrice)) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy new file mode 100644 index 000000000..fb9b88402 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15Export.groovy @@ -0,0 +1,158 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15Export { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService districtExecutorService = Executors.newFixedThreadPool(30) + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["扬州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + CountDownLatch latch = new CountDownLatch(cities.size()) + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + executorService.execute(new Runnable() { + @Override + void run() { + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + // 小区可能存在多个别名 + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + districtExecutorService.execute(new Runnable() { + @Override + void run() { + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + // 不可用append,会被覆盖 + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + +// DBCursor dbCursor = collection.find(new BasicDBObject("city", city).append("region", objOur.region).append("district_name", objOur.name) +// .append("weekend", new BasicDBObject('$lte', nextWeekDate)) +// .append("weekend", new BasicDBObject('$gte', firstWeekDate))).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + }) + }; + districtLatch.await() + row[6] = totalCount.get() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + }; + latch.await() + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + def row = [] + row[0] = city + row[6] = 0 + row[7] = 0 + result << row + } + } + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + executorService.shutdown() + districtExecutorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy new file mode 100644 index 000000000..40a866be1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/PPE15ExportTest.groovy @@ -0,0 +1,130 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCursor +import org.apache.commons.lang3.time.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class PPE15ExportTest { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + + static main(args) { +// def cities = ["北京", "上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞" ,"佛山","杭州", +// "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", +// "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", +// "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", +// "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", +// "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", +// "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", +// "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + def cities = ["常州"] + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + for (def cityTemp : cities) { + final city = cityTemp + println("当前处理城市:" + city) + try { + AtomicInteger totalCount = new AtomicInteger(0) + AtomicInteger count = new AtomicInteger(0) + def row = [] + row[0] = city + // 小区 + final def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, obj) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, obj) + } + } + } + } + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + int ppe15Count = 0 + CountDownLatch districtLatch = new CountDownLatch(fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).count()) + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', com.fangjia.script.util.DateUtils.parseFromTightYYYYMMDD("2020-03-24")))) + .batchSize(10000).each { obj -> + try { + if (map.get(obj.region + obj.district_name)) { + def objOur = map.get(obj.region + obj.district_name) + Double avgPrice = obj.avg_price + Date tradeDate = obj.trade_date + // 最相近一周的该小区均价 + Date nextWeekDate = DateUtils.addDays(tradeDate, 7) + Date firstWeekDate = DateUtils.addDays(tradeDate, -7) + + com.mongodb.DBObject dbObject = new BasicDBObject("city", city) + dbObject.put("region", obj.region) + dbObject.put("district_name", objOur.name) + dbObject.put("weekend", new BasicDBObject('$lte', nextWeekDate).append('$gte', firstWeekDate)) + dbObject.put("region", obj.region) + DBCursor dbCursor = collection.find(dbObject).batchSize(10000).sort(new BasicDBObject("weekend", -1)).limit(1) + + Double weekendAvgPrice = 0 + dbCursor.each { o -> + weekendAvgPrice = o.avg_price + } + + if (weekendAvgPrice == 0) { + println("周均价未找到:" + city + " " + obj.region + " " + obj.district_name) + } else { + boolean ppe15 = Math.abs((weekendAvgPrice - avgPrice) * 100 / avgPrice) <= 15 + if (ppe15) { + count.incrementAndGet() + } + } + totalCount.incrementAndGet() + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + districtLatch.countDown() + } + } + districtLatch.await() + row[7] = count.get() + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(count.get() * 100) / Double.valueOf(totalCount.get())) + "%)" + retMap.put(city, row) + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + + } + }; + for (def city: cities) { + if (retMap.get(city)) { + result << retMap.get(city) + } else { + result << 0 + } + } + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy new file mode 100644 index 000000000..dbcf2139f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/RentExport.groovy @@ -0,0 +1,153 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 上海房屋租金导出 + * + * @author shenjian + * @since 2021/03/22 + */ +class RentExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia", "district_stat_rent", "192.168.0.43",28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static Map retMap = new ConcurrentHashMap() + + static main(args) { + def result = [] + result << ["城市","区域","小区名称","房型","租金(月)","计算方式", "经度", "纬度"] + + int count = DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).count() + AtomicInteger size = new AtomicInteger(count) + println("上海小区" + size.get()) + CountDownLatch latch = new CountDownLatch(count) + DBUtils.seaweed.find(new BasicDBObject("city", "上海"), + new BasicDBObject("city", 1).append("name", 1).append("region", 1).append("lng", 1).append("lat", 1)) + .batchSize(10000).each {obj -> + executorService.submit(new Runnable() { + @Override + void run() { + try { + DBObject query = new BasicDBObject() + query.append("city", "上海").append("district_name", obj.name).append("region", obj.region) + .append("end_month_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2020-09-22"))) + .append("house_type", new BasicDBObject('$lt', 5)) + + DBObject filter = new BasicDBObject() + filter.append("region", 1).append("city", 1).append("district_name", 1).append("avg_price", 1) + .append("house_type", 1).append("sample_enum", 1).append("_id", 0).append("end_month_date", 1) + + buildResult(query, filter, result, obj.lng, obj.lat) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + println("*********************" + size.decrementAndGet()) + } + } + }) + } + + latch.await() + result.addAll(retMap.values()) + com.fangjia.script.util.ExcelUtils.write("G:\\数据\\建行\\租金\\租金.xlsx", result, "租金"); + executorService.shutdown() + } + + static boolean buildResult (def query, def filter, def result, def lng, def lat) { + collection.find(query, filter).batchSize(10000).sort(new BasicDBObject("end_month_date", -1).append("house_type", 1)).each {obj -> + def row = [] + row[0] = obj.city + row[1] = obj.region + row[2] = obj.district_name + row[3] = obj.house_type + row[4] = obj.avg_price + row[5] = obj.sample_enum + row[6] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lng / 10000000000d) + row[7] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, lat / 10000000000d) + + if (!retMap.containsKey(obj.district_name + obj.region + obj.house_type)) { + // 判断价格是否合理 + if (isPriceReasonable(obj.district_name, obj.region, obj.house_type, obj.avg_price)) { + retMap.put(obj.district_name + obj.region + obj.house_type, row) + println(retMap.size()) + } else { + println("当前价格不合理,被过滤掉" + obj.district_name + obj.region + obj.house_type) + } + } + } + } + + static boolean isPriceReasonable(def district_name, def region, def house_type, def avg_price) { + if (retMap.contains(district_name + region + "1")) { + printf("一居室" + retMap.get(district_name + region + "1")[4]) + } + if (retMap.contains(district_name + region + "2")) { + printf("二居室" + retMap.get(district_name + region + "2")[4]) + } + if (retMap.contains(district_name + region + "3")) { + printf("三居室" + retMap.get(district_name + region + "3")[4]) + } + println("当前居室" + house_type + ", 价格为" + avg_price) + switch (house_type) { + case 2: + println(retMap.get(district_name + region + "1")) + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 3: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + case 4: + if (retMap.get(district_name + region + "1")) { + double oneAvgPrice = Double.valueOf(retMap.get(district_name + region + "1")[4]) + if (oneAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "2")) { + double twoAvgPrice = Double.valueOf(retMap.get(district_name + region + "2")[4]) + if (twoAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + if (retMap.get(district_name + region + "3")) { + double threeAvgPrice = Double.valueOf(retMap.get(district_name + region + "3")[4]) + if (threeAvgPrice.compareTo(Double.valueOf(avg_price)) > 0) { + return false + } + } + break; + } + return true + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy new file mode 100644 index 000000000..b0e7a120e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/TotalTestExport.groovy @@ -0,0 +1,92 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class TotalTestExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["贵阳"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区 + def map = [:] + DBUtils.seaweed.find( new BasicDBObject("city", city).append("cat", "district").append("visible", 0), new BasicDBObject("city", 1).append("region", 1).append("name", 1).append("alias", 1)) + .batchSize(10000).each {obj -> + if (obj.region != null) { + map.put(obj.region + obj.name, 1) + if (obj.alias) { + for (def a : obj.alias) { + map.put(obj.region + a, 1) + } + } + } + } + + + // 测试数据量(小区近一年成交量) + int tradeCount = 0 + fangjiaConn.find(new BasicDBObject("city", city).append("trade_date", new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)))) + .batchSize(10000).each { obj -> + if (map.get(obj.region + obj.district_name)) { + tradeCount++ + } else { + println("小区不位于小区表" + city + " " + obj.region + " " + obj.district_name) + } + } + row[6] = tradeCount + + // PPE15数量及占比(小区近一年成交的每条数据,找最相近一周的该小区均价) + //int ppe15 = + + //row[7] = + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("G:\\数据\\模型级联项目涉及城市v1.1-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy new file mode 100644 index 000000000..82f000f52 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/ValueRateCityInfoExport.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch + +/** + * 模型级联项目涉及城市 + * + * @author shenjian + * @since 2021/3/23 + */ +class ValueRateCityInfoExport { + + // 小区均价集合 + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "192.168.0.43",28017) + // 小区成交集合 + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43",28017) + static Map retMap = new ConcurrentHashMap() + // 在此更改时间 + static final def TRADE_DATE_START = "2020-03-28" + + static main(args) { + def cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州", + "合肥","南京","青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台", + "徐州","无锡","温州","太原","台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁", + "南昌","兰州","常州","大连","福州","贵阳","哈尔滨","海口","惠州","济南","嘉兴","金华", + "昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江","岳阳","银川","宜昌","盐城", + "芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳","柳州", + "连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口", + "西宁","日照","拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + //def cities = ["重庆","郑州", "长沙", "北京"] + + def result = [] + result << ["城市","小区数量", "可估值小区数量及占比", "有样本估值数量及占比", "无样本估值数量及占比", "可信度大于0.5的数量", "测试数据量", "PPE15数量及占比"] + + //CountDownLatch latch = new CountDownLatch(cities.size()) + for (def city : cities) { + try { + def row = [] + row[0] = city + // 小区数量 + int seaweedCount = DBUtils.seaweed.count( new BasicDBObject("city", city).append("cat", "district").append("visible", 0)) + + // 可估值小区数量 + int valuateCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 有样本估值数量及占比 + int haveSampleCount = collection.find(new BasicDBObject("city", city) + .append("value_type", 2) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + ).count() + + // 无样本估值数量及占比 + int notHaveSampleCount = valuateCount - haveSampleCount + + // 可信度大于0.5的数量 + int creditCount = collection.find(new BasicDBObject("city", city) + .append("weekend", DateUtils.parseFromTightYYYYMMDD(TRADE_DATE_START)) + .append("credit", new BasicDBObject('$gte', 0.5d)) + ).count() + + row[1] = seaweedCount + row[2] = valuateCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(valuateCount * 100) / Double.valueOf(seaweedCount)) + "%)" + if (valuateCount != 0) { + row[3] = haveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(haveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[4] = notHaveSampleCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(notHaveSampleCount * 100) / Double.valueOf(valuateCount)) + "%)" + row[5] = creditCount + "(" + com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(creditCount * 100) / Double.valueOf(valuateCount)) + "%)" + } + + result << row + println("当前处理城市" + city + " " + row) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + //latch.countDown() + } + } + //latch.await() + ExcelUtils.write("F:\\数据\\模型级联项目涉及城市v1.0-test.xlsx", result, "Sheet1") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy new file mode 100644 index 000000000..399b33c38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/CopyDataForAnnPricePre.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import org.apache.commons.lang3.time.DateUtils +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.* +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder + +import java.util.concurrent.atomic.AtomicLong + +/** + * 拷贝1.31前四周数据至3.28日缺失的四周数据中 + * 注意时间更新 + * + * @author shenjian + * @since 2021/3/30 + */ +class CopyDataForAnnPricePre { + + static RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + + static final long timestamp = (DateUtils.parseDate("2021-03-28", "yyyy-MM-dd").getTime() - + DateUtils.parseDate("2021-01-31", "yyyy-MM-dd").getTime()) + static final int days = timestamp / 1000 / 3600 / 24 + + static void main(String[] args) { + // 验证数据是否正确写入 + //String[] weeksNew= ["2021-03-07", "2021-03-14", "2021-03-21", "2021-03-28"] + String[] weeksNew= ["2021-03-07", "2021-01-31"] + for (String week : weeksNew) { + println(DateUtils.parseDate(week, "yyyy-MM-dd").getTime()) + } + //String[] weeksNew=["2021-03-28"] +// ///////////////////// 删除数据临时逻辑 +// // 先删除数据 +// DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("ann_price_pre") +// deleteRequest.setQuery(QueryBuilders.termQuery("date", DateUtils.parseDate("2021-03-07", "yyyy-MM-dd").getTime())) +// // 批次大小 +// deleteRequest.setBatchSize(1000); +// restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) +// ///////////////////// + checkNum(weeksNew) + + BulkRequest request = new BulkRequest(); + for (String week : weeksNew) { + AtomicLong count = new AtomicLong(0) + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.addDays(DateUtils.parseDate(week, "yyyy-MM-dd"), -days).getTime())) + .size(10000); + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L)); + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + .source(searchSourceBuilder) + .scroll(scroll); // 设置scroll间隔 + + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); //通过发送初始搜索请求来初始化搜索上下文 + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits() + write(count, searchHits, timestamp, request); + + while (searchHits != null && searchHits.length > 0) { //通过循环调用搜索滚动api来检索所有搜索命中,直到没有文档返回 + //处理返回的搜索结果 + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); //创建一个新的搜索滚动请求,保存最后返回的滚动标识符和滚动间隔 + scrollRequest.scroll(scroll); + searchResponse = restHighLevelClient.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + write(count, searchHits, timestamp, request); + } + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT) + println(week + "剩余批量操作成功:" + bulkResponse.status()) + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); //完成滚动后,清除滚动上下文 + clearScrollRequest.addScrollId(scrollId); + ClearScrollResponse clearScrollResponse = restHighLevelClient.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + boolean succeeded = clearScrollResponse.isSucceeded(); + println("清除滚动句柄" + succeeded) + } + checkNum(weeksNew) + + restHighLevelClient.close() + } + + static write(def count, def searchHits, def timestamp, def request) { + for (SearchHit hit : searchHits) { + Map map = hit.getSourceAsMap(); + map.put("date", Long.valueOf(map.get("date").toString()) + timestamp) + + request.add(new IndexRequest("ann_price_pre").source(map).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + if (count.get() >= 5000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + println("批量操作成功:" + bulkResponse.status()) + request = new BulkRequest(); + count = new AtomicLong(0) + } + } + } + + static checkNum(def weeksNew) { + // 验证数据是否正确写入 + for (String week : weeksNew) { + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(QueryBuilders.termQuery("date", DateUtils.parseDate(week, "yyyy-MM-dd").getTime())); + + SearchRequest searchRequest = new SearchRequest("ann_price_pre") + searchRequest.source(searchSourceBuilder) + SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT); + + println(searchResponse.status().name() + " " + searchResponse.getHits().totalHits) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy new file mode 100644 index 000000000..68d16920d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat.groovy @@ -0,0 +1,98 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.util.ElasticUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11,先写入mongo + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + QueryBuilder queryBuilder = QueryBuilders.matchAllQuery() + def districtPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) + //def newDistrictPriceStats = ElasticUtils.executeSimpleQuery("192.168.0.206", "district_price_stat", queryBuilder) +// RestHighLevelClient restHighLevelClient206 = getClient206() +// RestHighLevelClient restHighLevelClient = getClient() +// //def districtPriceStats +// GetResponse getResponse = getDocument(restHighLevelClient) + for (def map : districtPriceStats) { + DBObject dbObject = new BasicDBObject() + + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + fangjiaConn.save(dbObject) + +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.id(id).timeout(TimeValue.timeValueSeconds(1)) +// request.source(map) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); +// +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + //restHighLevelClient.close() + } + +// static RestHighLevelClient getClient() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.216",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static RestHighLevelClient getClient206() { +// RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost("192.168.0.206",9200,"http")) +// .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { +// @Override +// HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { +// httpClientBuilder.disableAuthCaching(); +// CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); +// credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); +// httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); +// return httpClientBuilder; +// } +// })) +// return restHighLevelClient +// } +// +// static GetResponse getDocument(RestHighLevelClient restHighLevelClient206) throws IOException { +// SearchRequest searchRequest = new SearchRequest("district_price_stat"); +// SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); +// //searchSourceBuilder.query(QueryBuilders.termQuery("url", "test")); +// searchRequest.source(searchSourceBuilder); +// SearchResponse response = restHighLevelClient206.search(searchRequest, RequestOptions.DEFAULT); +// System.out.println(response.getSourceAsString()); //获取文档的内容 +// return response +// } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy new file mode 100644 index 000000000..999f1eaf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/SyncDistrictPriceStat2.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.es + +import com.fangjia.core.util.MongoUtils +import com.fangjia.script.temp.shenjian.es.util.RestHighLevelClientUtils +import com.mongodb.DBCollection +import org.elasticsearch.action.DocWriteRequest +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.index.reindex.DeleteByQueryRequest + +import java.util.concurrent.atomic.AtomicLong + +/** + * 由于快照恢复字段类型错误 + * 同步es6.4该索引至es7.11 + * + * @author shenjian + * @since 2021/3/30 + */ +class SyncDistrictPriceStat2 { + + static DBCollection fangjiaConn = MongoUtils.getCollectionByName("test", "district_price_stat", "192.168.0.136", 27017) + + static void main(String[] args) { + RestHighLevelClient restHighLevelClient = RestHighLevelClientUtils.getClient() + // 先删除数据 + DeleteByQueryRequest deleteRequest = new DeleteByQueryRequest("district_price_stat") + deleteRequest.setQuery(QueryBuilders.matchAllQuery()) + // 批次大小 + deleteRequest.setBatchSize(1000); + restHighLevelClient.deleteByQuery(deleteRequest, RequestOptions.DEFAULT) + + BulkRequest request = new BulkRequest(); + AtomicLong count = new AtomicLong() + fangjiaConn.find().batchSize(10000).each {map -> + Map dbObject = new HashMap() + dbObject.put("badcase_count", map.badcase_count) + dbObject.put("city", map.city) + dbObject.put("city_grade", map.city_grade) + dbObject.put("credit_district_count", map.credit_district_count) + dbObject.put("date", map.date) + dbObject.put("district_count", map.district_count) + dbObject.put("price_district_count", map.price_district_count) + dbObject.put("score", map.score) + dbObject.put("vote_district", map.vote_district) + + request.add(new IndexRequest("district_price_stat").source(dbObject).opType(DocWriteRequest.OpType.CREATE)); + count.incrementAndGet() + // 每一千写入 + if (count.get() >= 1000) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + request = new BulkRequest(); + count = new AtomicLong(0) + } + + // 单个插入文档 +// // 创建索引请求 +// IndexRequest request = new IndexRequest("district_price_stat") +// request.timeout(TimeValue.timeValueSeconds(1)) +// request.source(dbObject) +// +// // 将请求发出去,获取响应结果 +// IndexResponse indexResponse = restHighLevelClient.index(request, RequestOptions.DEFAULT); + +// System.out.println(indexResponse.status()); // 返回当前操作的类型:CREATE +// System.out.println(indexResponse.toString()); //响应内容 + } + // 不足一千写入 + if (count.get() > 0) { + BulkResponse bulkResponse = restHighLevelClient.bulk(request, RequestOptions.DEFAULT); + } + restHighLevelClient.close() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy new file mode 100644 index 000000000..bbbe4ebf9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/es/util/RestHighLevelClientUtils.groovy @@ -0,0 +1,35 @@ +package com.fangjia.script.temp.shenjian.es.util + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient + +/** + * @author shenjian + * @since 2021/3/30 + */ +class RestHighLevelClientUtils { + + static final def HOST = "192.168.0.216" + + static RestHighLevelClient getClient() { + RestHighLevelClient restHighLevelClient = new RestHighLevelClient(RestClient.builder(new HttpHost(HOST,9200,"http")) + .setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) { + httpClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpClientBuilder; + } + })) + return restHighLevelClient + } +} From d6f7803c6cd27b7263a2b3e539b714289afceef3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Apr 2021 14:38:53 +0800 Subject: [PATCH 002/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=95=86=E5=9C=88?= =?UTF-8?q?=E5=9C=B0=E5=9D=80=E7=BB=8F=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/BusinessDianpingAddr.groovy | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy new file mode 100644 index 000000000..dd8719a3f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy @@ -0,0 +1,60 @@ +package com.fangjia.script.temp.shenjian + +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +/** + * 获取商圈地标 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingAddr { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + + static void main(String[] args) { + def cities = ["苏州", "西安", "合肥"] + for (def city : cities) { + DBObject query = new BasicDBObject() + query.put("city", city) + collection.find(query).batchSize(300).each {obj -> + def id = obj._id + if (obj.lng == null || obj.lat == null) { + // 查询坐标位置,并更新 + def location = this.parserByGaode(city, obj.address) + if (location != null) { + DBObject updateField = new BasicDBObject() + updateField.put("lng", location.lng) + updateField.put("lat", location.lat) + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', updateField), false, true) + } + } + } + } + collection.find() + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + if (str && ["门牌号", "单元号", "热点商圈", "兴趣点"].contains(level)) { + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } else { + println address + " " + level + return null + } + } + } + } +} From a4717f4e83b7f0a8a1d015f119278bbc6185094b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 13 Apr 2021 19:50:58 +0800 Subject: [PATCH 003/212] =?UTF-8?q?=E5=95=86=E5=9C=88=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../BusinessDianpingAddr.groovy | 4 +- .../business/BusinessDianpingExport.groovy | 95 +++++++++++++++++++ .../business/BusinessDianpingModify.groovy | 29 ++++++ 3 files changed, 125 insertions(+), 3 deletions(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{ => business}/BusinessDianpingAddr.groovy (95%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy similarity index 95% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy index dd8719a3f..934c3411d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/BusinessDianpingAddr.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy @@ -1,4 +1,4 @@ -package com.fangjia.script.temp.shenjian +package com.fangjia.script.temp.shenjian.business import com.fangjia.script.util.MongoUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils @@ -6,7 +6,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import net.sf.json.JSONObject -import org.bson.types.ObjectId /** * 获取商圈地标 @@ -37,7 +36,6 @@ class BusinessDianpingAddr { } } } - collection.find() } static def parserByGaode(def city, def address) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy new file mode 100644 index 000000000..b18a14517 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -0,0 +1,95 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈地标导出 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingExport { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + static DBCollection cityCollection = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def cities = ["苏州", "西安", "合肥"] + def headers = ["城市", "区域", "商圈名称", "地标名称(3-5个)", "地标地址(与地标名称对应)", "地标经度(与地标名称对应)", "地标纬度(与地标名称对应)", "备注", "创建时间", "更新时间", "来源公司"]; + for (int i = 0; i < cities.size(); i++) { + final def city = cities[i]; + def result = [] + // 获取所有商圈信息 + def businessNames = collection.distinct("name", new BasicDBObject("city", city)) + // 获取商圈下3个地标 + for (def name : businessNames) { + List landmarks = new ArrayList<>() + List addresses = new ArrayList<>() + List lngs = new ArrayList<>() + List lats = new ArrayList<>() + String region = "" + String c_date + String m_date + int landmarkLen = 0; + collection.find(new BasicDBObject("city", city).append("name", name)).sort(new BasicDBObject("order", 1)).each { obj -> + // 按人气取经纬度不为空的地标 + if (landmarks.size() < 5) { + // 特殊标题去重 + if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1 + || String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1 + || String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1 + || String.valueOf(obj.landmark).indexOf(",") != -1) { + + } else { + // 防止导入导致字段过长,一个字符2个字节 + if (landmarkLen + String.valueOf(obj.landmark).length() <= 100) { + landmarkLen += String.valueOf(obj.landmark).length() + landmarks.add(obj.landmark) + addresses.add(obj.address) + lngs.add(obj.lng) + lats.add(obj.lat) + + if (StringUtils.isBlank(region)) { + region = obj.region + c_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.c_date, "yyyy-MM-dd HH:mm:ss"); + m_date = com.fangjia.script.util.DateUtil.formatByPattern(obj.m_date, "yyyy-MM-dd HH:mm:ss"); + } + } + } + } + } + def ret = [] + def city_code + def district_code + if (StringUtils.isNotBlank(region)) { + cityCollection.find(new BasicDBObject("fangjia_city", city).append("district", new BasicDBObject('$regex', region + "+")), + new BasicDBObject("city_code", 1).append("district_code", 1)).each {obj -> + city_code = obj.city_code + district_code = obj.district_code + } + } + ret[0] = city_code + ret[1] = district_code + ret[2] = name + ret[3] = StringUtils.join(landmarks, ",") + ret[4] = StringUtils.join(addresses, ",") + ret[5] = StringUtils.join(lngs, ",") + ret[6] = StringUtils.join(lats, ",") + ret[7] = "" + ret[8] = c_date + ret[9] = m_date + ret[10] = "fjw" + if (district_code != null) { + result << ret + } + } + FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy new file mode 100644 index 000000000..dd0c4c2dc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.business + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * 商圈数据修正 + * + * @author shenjian + * @since 2021/4/13 + */ +class BusinessDianpingModify { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + + static void main(String[] args) { + def cityCodes = ["320500","610100","340100"]; + def city = "合肥" + def origin = "政务区" + def now = "蜀山区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + } +} From 77aa3327e296a540f655d935264c7f6bac0d77c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Apr 2021 14:47:34 +0800 Subject: [PATCH 004/212] =?UTF-8?q?=E5=9C=B0=E5=9D=80=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E5=AE=8C=E5=96=84=E4=B8=8E=E7=A9=BA=E6=A0=BC=E5=A4=84?= =?UTF-8?q?=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/business/BusinessDianpingAddr.groovy | 11 +++-------- .../shenjian/business/BusinessDianpingExport.groovy | 7 ++++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy index 934c3411d..9e3d9b54e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingAddr.groovy @@ -26,7 +26,7 @@ class BusinessDianpingAddr { def id = obj._id if (obj.lng == null || obj.lat == null) { // 查询坐标位置,并更新 - def location = this.parserByGaode(city, obj.address) + def location = this.parserByGaode(city, obj.region + obj.name + obj.address) if (location != null) { DBObject updateField = new BasicDBObject() updateField.put("lng", location.lng) @@ -45,13 +45,8 @@ class BusinessDianpingAddr { if (ret) { def str = ret.get(0).location def level = ret.get(0).level - if (str && ["门牌号", "单元号", "热点商圈", "兴趣点"].contains(level)) { - def arrs = str.split(",") - return ["lng": arrs[0], "lat": arrs[1]] - } else { - println address + " " + level - return null - } + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy index b18a14517..dca87e75e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -48,9 +48,10 @@ class BusinessDianpingExport { } else { // 防止导入导致字段过长,一个字符2个字节 - if (landmarkLen + String.valueOf(obj.landmark).length() <= 100) { - landmarkLen += String.valueOf(obj.landmark).length() - landmarks.add(obj.landmark) + def landmark = String.valueOf(obj.landmark).replaceAll(" ", "") + if (landmarkLen + String.valueOf(landmark).length() <= 100) { + landmarkLen += String.valueOf(landmark).length() + landmarks.add(landmark) addresses.add(obj.address) lngs.add(obj.lng) lats.add(obj.lat) From 5b75537604009c717ee81a5adb2002f932e5dca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 14 Apr 2021 18:42:09 +0800 Subject: [PATCH 005/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=95=86=E5=9C=88?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../business/BusinessDianpingExport.groovy | 6 +++-- .../business/BusinessDianpingModify.groovy | 22 +++++++++++++++++++ 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy index dca87e75e..3f03c1c01 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingExport.groovy @@ -44,12 +44,12 @@ class BusinessDianpingExport { if (String.valueOf(obj.landmark).indexOf("│") != -1 || String.valueOf(obj.landmark).indexOf("/") != -1 || String.valueOf(obj.landmark).indexOf("|") != -1 || String.valueOf(obj.landmark).indexOf("#") != -1 || String.valueOf(obj.landmark).indexOf("地铁") != -1 || String.valueOf(obj.landmark).indexOf("!") != -1 || String.valueOf(obj.landmark).indexOf("、") != -1 || String.valueOf(obj.landmark).indexOf(")") != -1 - || String.valueOf(obj.landmark).indexOf(",") != -1) { + || String.valueOf(obj.landmark).indexOf(",") != -1 || String.valueOf(obj.landmark).indexOf("商圈附近") != -1 || String.valueOf(obj.landmark).indexOf("】") != -1) { } else { // 防止导入导致字段过长,一个字符2个字节 def landmark = String.valueOf(obj.landmark).replaceAll(" ", "") - if (landmarkLen + String.valueOf(landmark).length() <= 100) { + if (landmark.length() <= 30 && landmarkLen + String.valueOf(landmark).length() <= 100) { landmarkLen += String.valueOf(landmark).length() landmarks.add(landmark) addresses.add(obj.address) @@ -88,6 +88,8 @@ class BusinessDianpingExport { ret[10] = "fjw" if (district_code != null) { result << ret + } else { + println("未找到商圈地标:" + name) } } FileUtils.batchCreateFile(result, "f:\\数据\\建行\\商圈\\bigdata_business_" + cityCodes[i] + ".dat", headers, "utf-8") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy index dd0c4c2dc..df968e183 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/business/BusinessDianpingModify.groovy @@ -25,5 +25,27 @@ class BusinessDianpingModify { collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) } + + city = "苏州" + origin = "工业园区" + now = "吴中区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + city = "苏州" + origin = "甪直" + now = "吴中区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + + city = "西安" + origin = "高新区" + now = "雁塔区" + collection.find(new BasicDBObject("city", city).append("region", origin)).batchSize(1000).each {obj -> + collection.update(new BasicDBObject("url", obj.url), new BasicDBObject('$set', new BasicDBObject("region", now)), false, true) + } + } } From 01e5dff436d96798c84af593db093fe65d8dfd23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 15 Apr 2021 14:16:36 +0800 Subject: [PATCH 006/212] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=88=AB=E5=90=8D?= =?UTF-8?q?=E8=A6=86=E7=9B=96=E7=8E=87=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../seaweed/SeaweedAliasExport.groovy | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy new file mode 100644 index 000000000..f8c9beb67 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedAliasExport.groovy @@ -0,0 +1,63 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 小区别名数量统计 + * + * @author shenjian + * @since 2021/4/15 + */ +class SeaweedAliasExport { + + static void main(String[] args) { + def cities = ["重庆","苏州","昆明","柳州","曲靖","三明","达州","石狮","大理","福安","平潭","迪庆州"] + def result = [] + result << ["城市", "小区数量", "别名数量", "别名覆盖率"] + for (def city : cities) { + def ret = [] + + DBObject query = new BasicDBObject(); + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) + + //def seaweedCount = DBUtils.seaweed.find(query).batchSize(5000).count() + + int seaweedCount = 0; + int aliasCount = 0; + + if (city == "平潭") { + query.put("city", "福州") + query.put("region", city) + } + if (city == "福安") { + query.put("city", "宁德") + query.put("region", city) + } + if (city == "石狮") { + query.put("city", "泉州") + query.put("region", city) + } + DBUtils.seaweed.find(query).batchSize(5000).each {obj -> + if (obj.alias) { + aliasCount++ + } + seaweedCount++ + } + + ret[0] = city + ret[1] = seaweedCount + ret[2] = aliasCount + if (seaweedCount > 0) { + ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(aliasCount) * 100 / Double.valueOf(seaweedCount)) + } + result << ret + } + println(result) + ExcelUtils.write("F:\\数据\\小区别名覆盖率.xlsx", result, "统计") + } +} From 4b07cf082fab3ccd68c2f47fe2e1bfdba326e4ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 15 Apr 2021 14:21:23 +0800 Subject: [PATCH 007/212] =?UTF-8?q?easyexcel=E5=BC=95=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index bb7959032..1a14ef3ee 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -476,6 +476,11 @@ aliyun-java-sdk-address-purification 1.0.0 + + com.alibaba + easyexcel + 2.2.8 + From d76d663419b5a44f709848d4dba903e41c5358ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 16 Apr 2021 17:22:07 +0800 Subject: [PATCH 008/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/school/DistanceDeal.groovy | 86 +++++++++++++++++++ .../shenjian/school/FillMidSchoolInfo.groovy | 59 +++++++++++++ .../FillMidSchoolWithPrimarySchool.groovy | 56 ++++++++++++ 3 files changed, 201 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy new file mode 100644 index 000000000..001bfd9b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.core.util.AddressLatlng +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +/** + * 处理小区-学校距离过大的数据 + * + * @author shenjian + * @since 2021/4/16 + */ +class DistanceDeal { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) + + // 修复 鑫苑国际城市花园 https://guojichengshihuayuanszxy.fang.com 区域问题 + static void main(String[] args) { +// def result = GaoDeApiUtils.geocode("120.701271,31.32798") +// print(result) + + /* def ret = AddressLatlng.getLngLatByAddress("苏州", "苏州工业园区星湾学校(西校区)") + print(ret)*/ + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", "120.68764") +// updateField.put("school_lat", "31.330963") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true) + + DBObject updateField = new BasicDBObject() + updateField.put("school_lng", 120.68764) + updateField.put("school_lat", 31.330963) + updateField.put("school_name", "星湾学校西校区") + updateField.put("school_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") + updateField.put("gaode_name", "星湾学校西校区") + updateField.put("gaode_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") + + collection.update(new BasicDBObject("match_name", new BasicDBObject('$in', ["星海人家", "新加花园", "中央景城", "加城湖滨公寓", "沁苑小区", "湖畔花园", + "中茵皇冠国际", "天域三期", "天域", "天域二期"])).append("match_region", "吴中"), new BasicDBObject('$set', updateField), false, true) + +// def location = this.parserByGaode("苏州", "苏州工业园区星湾学校(西校区)") +// print(location) + +// if (location != null) { + //DBObject updateField = new BasicDBObject() +// updateField.put("match_lng", "120.614203") +// updateField.put("match_lat", "31.309876") +// collection.update(new BasicDBObject("url", "https://jingdelu0512.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "120.610651479") +// updateField.put("match_lat", "31.3131018333") +// collection.update(new BasicDBObject("url", "https://wuqufang.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "117.2703018188") +// updateField.put("match_lat", "31.7484989166") +// collection.update(new BasicDBObject("url", "https://binghuqiancheng.fang.com"), new BasicDBObject('$set', updateField), false, true) + +// +// updateField.put("match_lng", "120.610343") +// updateField.put("match_lat", "31.312715") +// collection.update(new BasicDBObject("url", "https://baolinsiqian.fang.com"), new BasicDBObject('$set', updateField), false, true) +// } +// DBObject updateField = new BasicDBObject() +// updateField.put("match_region", "蜀山") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10689.htm"), new BasicDBObject('$set', updateField), false, true) +// } + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy new file mode 100644 index 000000000..b621eb9fd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolInfo { + + static void main(String[] args) { + Map> map = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + String districtName = String.valueOf(data.get(2)) + if (districtName.contains("期")) { + String key = districtName.substring(0, districtName.length() - 2); + if (data.get(11) != null) { // 存在对口初中 + map.put(key, data); + } + } + } + print(map.size()) + Long count = 0; + for (List data : list) { + String districtName = String.valueOf(data.get(2)) + String key + if (districtName.contains("期")) { + key = districtName.substring(0, districtName.length() - 2) + } else { + key = districtName + } + if (map.get(key) != null && data.get(11) == null) { + List val = map.get(key) + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + count++ + println(data) + } + } + + print("优化数量" + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + list.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy new file mode 100644 index 000000000..b67c5a375 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolWithPrimarySchool { + + static void main(String[] args) { + Map> map = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + // 对口小学 对口中学 + String city = data.get(0) + String region = data.get(1) + if (data.get(7) != null && data.get(11) != null) { + map.put(city + region + data.get(7), data) + } + } + print(map.size()) + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String primarySchool = data.get(7) + + String key = city + region + primarySchool + if (map.get(key) != null && data.get(11) == null) { + List val = map.get(key) + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + count++ + println(data) + } + } + + print("优化数量" + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + list.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据") + } +} From 8cad854b3a337b489f660cb92040dc2ef43d24da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 16 Apr 2021 17:22:07 +0800 Subject: [PATCH 009/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E8=A1=A5=E5=85=85=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/school/DistanceDeal.groovy | 90 ++++++++++++++++ .../shenjian/school/FillMidSchoolInfo.groovy | 101 ++++++++++++++++++ .../FillMidSchoolWithPrimarySchool.groovy | 57 ++++++++++ .../temp/shenjian/school/Standard.groovy | 67 ++++++++++++ 4 files changed, 315 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy new file mode 100644 index 000000000..d19524f07 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -0,0 +1,90 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.core.util.AddressLatlng +import com.fangjia.script.district.Distance +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.bson.types.ObjectId + +/** + * 处理小区-学校距离过大的数据 + * + * @author shenjian + * @since 2021/4/16 + */ +class DistanceDeal { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) + + // 修复 鑫苑国际城市花园 https://guojichengshihuayuanszxy.fang.com 区域问题 + static void main(String[] args) { + def result = GaoDeApiUtils.geocode("117.363998413086,31.89208984375") + print(result) + + /* def ret = AddressLatlng.getLngLatByAddress("苏州", "苏州工业园区星湾学校(西校区)") + print(ret)*/ + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", "120.68764") +// updateField.put("school_lat", "31.330963") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true) + +// double length = Distance.GetDistance(); +// print(length) + +// DBObject updateField = new BasicDBObject() +// updateField.put("school_lng", 120.68764) +// updateField.put("school_lat", 31.330963) +// updateField.put("school_name", "星湾学校西校区") +// updateField.put("school_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") +// updateField.put("gaode_name", "星湾学校西校区") +// updateField.put("gaode_address", "苏州市吴中区吴中区园区苏州工业园区直属镇苏州工业园区槟榔路湖畔花园边上") +// +// collection.update(new BasicDBObject("match_name", new BasicDBObject('$in', ["星海人家", "新加花园", "中央景城", "加城湖滨公寓", "沁苑小区", "湖畔花园", +// "中茵皇冠国际", "天域三期", "天域", "天域二期"])).append("match_region", "吴中"), new BasicDBObject('$set', updateField), false, true) + +// def location = this.parserByGaode("苏州", "苏州工业园区星湾学校(西校区)") +// print(location) + +// if (location != null) { + //DBObject updateField = new BasicDBObject() +// updateField.put("match_lng", "120.614203") +// updateField.put("match_lat", "31.309876") +// collection.update(new BasicDBObject("url", "https://jingdelu0512.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "120.610651479") +// updateField.put("match_lat", "31.3131018333") +// collection.update(new BasicDBObject("url", "https://wuqufang.fang.com"), new BasicDBObject('$set', updateField), false, true) +// +// updateField.put("match_lng", "117.2703018188") +// updateField.put("match_lat", "31.7484989166") +// collection.update(new BasicDBObject("url", "https://binghuqiancheng.fang.com"), new BasicDBObject('$set', updateField), false, true) + +// +// updateField.put("match_lng", "120.610343") +// updateField.put("match_lat", "31.312715") +// collection.update(new BasicDBObject("url", "https://baolinsiqian.fang.com"), new BasicDBObject('$set', updateField), false, true) +// } +// DBObject updateField = new BasicDBObject() +// updateField.put("match_region", "蜀山") +// collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10689.htm"), new BasicDBObject('$set', updateField), false, true) +// } + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy new file mode 100644 index 000000000..95c48476d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolInfo.groovy @@ -0,0 +1,101 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang3.StringUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolInfo { + + static void main(String[] args) { + Map> map = new HashMap<>(); + Map> map2 = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String districtName = String.valueOf(data.get(2)) + String key = city + region + if (districtName.contains("期")) { + key += districtName.substring(0, districtName.length() - 2); + if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中 + map.put(key, data); + } + if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学 + map2.put(key, data); + } + } else { + key += districtName + if (StringUtils.isNotBlank(data.get(11))) { // 存在对口初中 + map.put(key, data); + } + if (StringUtils.isNotBlank(data.get(6))) { // 存在对口小学 + map2.put(key, data); + } + } + } + print(map.size()) + Long countPrimary = 0; + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String districtName = String.valueOf(data.get(2)) + String key = city + region + if (districtName.contains("期")) { + key += districtName.substring(0, districtName.length() - 2) + } else { + key += districtName + } + if (map.get(key) != null) { + List val = map.get(key) + if (StringUtils.isBlank(data.get(11))) { + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + + count++ + println("初中" + data) + } + } + if (map2.get(key) != null) { + List val = map2.get(key) + if (StringUtils.isBlank(data.get(6))) { + data.set(6, val.get(6)) + data.set(7, val.get(7)) + data.set(8, val.get(8)) + data.set(9, val.get(9)) + data.set(10, val.get(10)) + + countPrimary++ + println("小学" + data) + } + } + } + + List> result = new ArrayList<>(); + for (List data : list) { + if (StringUtils.isNotBlank(data.get(6))) { + result.add(data) + } + } + + print("优化数量" + countPrimary + " " + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + result.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", result, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy new file mode 100644 index 000000000..8f4ace4c8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/FillMidSchoolWithPrimarySchool.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang3.StringUtils + +/** + * 填充规则 + * + * 1.中学信息按期次填充 + * + * 2.A小区对口小学:长安区第一小学、 对口中学:长安区第二中学。 B 小区对口小区也是长安区第一小学,但是B小区的对口中学是空的额。 + * 但是B小区和A小区的对口小学一样,那么我们就默认B小区的对口中学也和A小区的对口中学一样 + * + * @author shenjian + * @since 2021/4/16 + */ +class FillMidSchoolWithPrimarySchool { + + static void main(String[] args) { + Map> map = new HashMap<>(); + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx") + for (List data : list) { + // 对口小学 对口中学 + String city = data.get(0) + String region = data.get(1) + if (org.apache.commons.lang.StringUtils.isNotBlank(data.get(7)) && StringUtils.isNotBlank(data.get(11))) { + map.put(city + region + data.get(7), data) + } + } + print(map.size()) + Long count = 0; + for (List data : list) { + String city = data.get(0) + String region = data.get(1) + String primarySchool = data.get(7) + + String key = city + region + primarySchool + if (map.get(key) != null && StringUtils.isBlank(data.get(11))) { + List val = map.get(key) + data.set(11, val.get(11)) + data.set(12, val.get(12)) + data.set(13, val.get(13)) + data.set(14, val.get(14)) + data.set(15, val.get(15)) + count++ + println(data) + } + } + + print("优化数量" + count) + String[] header = ["城市","区域","小区名称","小区经度","小区纬度","小区地址","对口小学","小学地址","小学属性","小学经度","小学纬度","对口中学","中学地址","中学属性","中学经度","中学纬度","备注","创建时间","修改时间","来源网站", + "小区ID","数据类型"]; + List headerList = Arrays.asList(header) + list.add(0, headerList) + ExcelUtils.write("F:\\工作\\数据采购\\楼盘对口学校V1.xlsx", list, "数据") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy new file mode 100644 index 000000000..989f77bc6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/Standard.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.school + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.ExcelUtils + +/** + * 标准化文件输出 + * + * @author shenjian + * @since 2021/4/16 + */ +class Standard { + + static void main(String[] args) { + List> list = ExcelUtils.read("F:\\工作\\数据采购\\楼盘对口学校V2.0.xlsx") + def resultOne = [] + def resultTwo = [] + def resultThree = [] + for (List data : list) { + // "320500","610100","340100" + if (data.get(0) == "320500") { + saveData(resultOne, data) + } else if (data.get(0) == "610000") { + data.set(0, 610100) + saveData(resultTwo, data) + } else if (data.get(0) == "340100") { + saveData(resultThree, data) + } + } + def headers = ["城市","区域","小区名称","小区经度","小区纬度", "小区地址","对口小学","小学地址","小学属性","小学批次","小学经度","小学纬度","对口中学","中学地址","中学属性", + "中学批次","中学经度","中学纬度","备注","创建时间","修改时间","来源网站"] + FileUtils.batchCreateFile(resultOne, "F:\\工作\\数据采购\\学校\\bigdata_commschool_320500.dat", headers, "utf-8") + FileUtils.batchCreateFile(resultTwo, "F:\\工作\\数据采购\\学校\\bigdata_commschool_610100.dat", headers, "utf-8") + FileUtils.batchCreateFile(resultThree, "F:\\工作\\数据采购\\学校\\bigdata_commschool_340100.dat", headers, "utf-8") + } + + static void saveData(def result, def data) { + def ret = [] + ret[0] = data.get(0) + ret[1] = data.get(1) + ret[2] = data.get(2) + ret[3] = data.get(3) != null && data.get(3) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(3))) : "" + + ret[4] = data.get(4) != null && data.get(4) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(4))) : "" + ret[5] = data.get(5) + + ret[6] = data.get(6) + ret[7] = data.get(7) + ret[8] = data.get(8) + ret[9] = "" + ret[10] = data.get(9) != null && data.get(9) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(9))) : "" + ret[11] = data.get(10) != null && data.get(10) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(10))) : "" + ret[12] = data.get(11) + ret[13] = data.get(12) + ret[14] = data.get(13) + ret[15] = "" + ret[16] = data.get(14) != null && data.get(14) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(14))) : "" + ret[17] = data.get(15) != null && data.get(15) != "" ? DataHandleUtils.getDoubleRound(6, Double.valueOf(data.get(15))) : "" + ret[18] = data.get(16) + ret[19] = data.get(17) + ret[20] = data.get(18) + ret[21] = data.get(19) + + result << ret + } +} From f1e7cc1d442c4a2278297cbc430542a3aaddd785 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 21 Apr 2021 14:39:40 +0800 Subject: [PATCH 010/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=88=BF=E5=A4=A9?= =?UTF-8?q?=E4=B8=8B=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/CommFang.groovy | 54 +++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy new file mode 100644 index 000000000..5da9281bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -0,0 +1,54 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 贝壳找房小区种子生成 + */ +class CommFang { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source","搜房") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.district_id + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = url.substring(url.lastIndexOf("/") + 1, url.lastIndexOf(".")) + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", "fang.com") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + set.add(id) + doc.add(dbObject) + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file From 040bbc5eca954da5d1b3c8726189b6f6f6214dab Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Wed, 21 Apr 2021 15:21:34 +0800 Subject: [PATCH 011/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E5=B0=8F=E5=8C=BA=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/CommFang.groovy | 2 +- .../script/temp/shenjian/crawl/CommKe.groovy | 57 +++++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy index 5da9281bd..a36b31539 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -6,7 +6,7 @@ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.time.DateUtils /** - * 贝壳找房小区种子生成 + * 房天下小区种子生成 */ class CommFang { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy new file mode 100644 index 000000000..0e2baf552 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy @@ -0,0 +1,57 @@ +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.apache.commons.lang3.time.DateUtils + +/** + * 贝壳找房小区种子生成 + */ +class CommKe { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static final def source = "贝壳找房" + static final def root = "ke.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("url", 1) + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(1000).each { obj -> + def url = obj.url + def cityShort = url.substring(url.indexOf("https://") + 8, url.indexOf(".")) + def id = obj.district_id + if (!set.contains(id) && StringUtils.isNumeric(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + set.add(id) + doc.add(dbObject) + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } +} \ No newline at end of file From a2081cf9df8600e73b8bc474b330f029ff6f6a8e Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Tue, 27 Apr 2021 09:32:35 +0800 Subject: [PATCH 012/212] =?UTF-8?q?=E6=97=A0=E6=95=88=E7=A7=8D=E5=AD=90?= =?UTF-8?q?=E6=B8=85=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/crawl/CommFang.groovy | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy index a36b31539..e44632fc2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -39,8 +39,10 @@ class CommFang { dbObject.put("c_date", c_date) dbObject.put("m_date", c_date) - set.add(id) - doc.add(dbObject) + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } if (doc.size() == 5000) { districtSeeds.insert(doc) doc.clear() From 7d05160e818177be19bfcbda28db1ec5abe9ed94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 30 Apr 2021 10:38:34 +0800 Subject: [PATCH 013/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E7=A6=8F=E5=B7=9E?= =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/crawl/Comm58.groovy | 68 +++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy new file mode 100644 index 000000000..3aa674810 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy @@ -0,0 +1,68 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.time.DateUtils + +/** + * @author shenjian + * @since 2021/4/30 + */ +class Comm58 { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static final def source = "58同城" + static final def root = "58.com" + + static void main(String[] args) { + Date c_date = new Date() + DBObject query = new BasicDBObject() + query.put("source",source) + query.put("city","福州") + query.put("c_date", new BasicDBObject('$gte', DateUtils.parseDate("2021-01-01", "yyyy-MM-dd"))); + DBObject filter = new BasicDBObject() + filter.put("district_id", 1) + filter.put("city", 1) + filter.put("region", 1) + Set set = new HashSet<>() + List doc = new ArrayList() + collection.find(query, filter).batchSize(10000).each { obj -> + try { + def url = obj.district_id + def cityShort = "fz" + + String[] info = url.split("/") + def id = info[info.size() - 1] + if (!set.contains(id)) { + DBObject dbObject = new BasicDBObject() + dbObject.put("city", obj.city) + dbObject.put("region", obj.region) + dbObject.put("city_short", cityShort) + dbObject.put("district_id", id) + dbObject.put("root", root) + dbObject.put("type", "comm") + dbObject.put("c_date", c_date) + dbObject.put("m_date", c_date) + + if (obj.region != null) { + set.add(id) + doc.add(dbObject) + } + if (doc.size() == 5000) { + districtSeeds.insert(doc) + doc.clear() + } + } + } catch(Exception ex) { + println("解析异常:" + obj) + } + } + districtSeeds.insert(doc) + doc.clear() + print("操作完成") + } + +} From 4c3d27236c8e8dc88dbe98d093d7d278cd0d6f8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 30 Apr 2021 10:41:27 +0800 Subject: [PATCH 014/212] =?UTF-8?q?=E7=A7=8D=E5=AD=90=E9=9B=86=E5=90=88?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/crawl/Comm58.groovy | 3 +-- .../com/fangjia/script/temp/shenjian/crawl/CommFang.groovy | 2 +- .../com/fangjia/script/temp/shenjian/crawl/CommKe.groovy | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy index 3aa674810..caf289056 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy @@ -13,7 +13,7 @@ import org.apache.commons.lang3.time.DateUtils class Comm58 { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) - static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) static final def source = "58同城" static final def root = "58.com" @@ -43,7 +43,6 @@ class Comm58 { dbObject.put("city_short", cityShort) dbObject.put("district_id", id) dbObject.put("root", root) - dbObject.put("type", "comm") dbObject.put("c_date", c_date) dbObject.put("m_date", c_date) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy index e44632fc2..46659dc54 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommFang.groovy @@ -11,7 +11,7 @@ import org.apache.commons.lang3.time.DateUtils class CommFang { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) - static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) static void main(String[] args) { Date c_date = new Date() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy index 0e2baf552..0cf014627 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/CommKe.groovy @@ -11,7 +11,7 @@ import org.apache.commons.lang3.time.DateUtils class CommKe { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) - static DBCollection districtSeeds = MongoUtils.getCollectionByName("fangjia_craw", "district_seeds", "192.168.0.235", 27018) + static DBCollection districtSeeds = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) static final def source = "贝壳找房" static final def root = "ke.com" From dfd5cf1e51ebc27a844f66f26faf9d361561617b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 8 May 2021 15:17:41 +0800 Subject: [PATCH 015/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=92=89=E9=92=89?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 12 +- .../shenjian/statistics/DingTalkUtils.groovy | 14 + .../temp/shenjian/statistics/Export.groovy | 51 +++ .../fangjia/script/statistics/ApiUtil.java | 305 ++++++++++++++++++ .../fangjia/script/statistics/Project.java | 92 ++++++ .../script/statistics/ProjectLabel.java | 57 ++++ .../com/fangjia/script/statistics/Task.java | 64 ++++ .../fangjia/script/statistics/TaskDetail.java | 67 ++++ .../fangjia/script/statistics/json/label.json | 38 +++ .../script/statistics/json/project.json | 225 +++++++++++++ .../fangjia/script/statistics/json/task.json | 85 +++++ .../script/statistics/json/taskDetail.json | 29 ++ 12 files changed, 1033 insertions(+), 6 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 1a14ef3ee..0d5adfa99 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -481,11 +481,11 @@ easyexcel 2.2.8 + + org.projectlombok + lombok + 1.18.20 + provided + - - - elastic.co - https://artifacts.elastic.co/maven - - diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy new file mode 100644 index 000000000..f49025b41 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy @@ -0,0 +1,14 @@ +/** + * 钉钉接口调用工具类 + * + * @author shenjian + * @since 2021/5/8 + */ +class DingTalkUtils { + + final String key = "ding0243a5c51f1f4aa8"; + + static void main(String[] args) { + + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy new file mode 100644 index 000000000..7f53ec256 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy @@ -0,0 +1,51 @@ +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel + +/** + * 绩效相关信息导出 + * + * @author shenjian + * @since 2021/5/8 + */ +class Export { + + static void main(String[] args) { + def result = [] + result << ['统计时间', '2021年4月1日-2021年4月30日'] + result << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] + + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[1] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[2] == null) { + ret[2] = label.getName() + } else { + ret[2] = ret[2] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + result << ret + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "项目统计"); + } +} + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java new file mode 100644 index 000000000..1c2836659 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -0,0 +1,305 @@ +package com.fangjia.script.statistics; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +import javax.crypto.Mac; +import javax.crypto.spec.SecretKeySpec; + +import com.alibaba.fastjson.JSON; +import com.alibaba.fastjson.JSONArray; +import org.apache.commons.codec.binary.Base64; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.HttpClient; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.DefaultHttpClient; + +import com.alibaba.fastjson.JSONObject; + +/** + * 铓傝殎鍒嗗伐寮�鏀惧钩鍙板伐鍏风被 + * 鐢变簬groovy璋冪敤浼氳鎶ョ鍚嶉敊璇棶棰橈紝鏁呯敤java鏂瑰紡 + */ +public class ApiUtil { + + private static final String secretKey = "HtRc7R5jI0plOOAukES4pM6yxWJbU9GyyM1xi4kH"; + private static final String corpId = "ding0243a5c51f1f4aa8"; + + public static String getApiSingnature(String body, Long timeStamp, String secretKey) throws Exception { + JSONObject bodyObj = JSONObject.parseObject(body); + return getApiSingnature(bodyObj, timeStamp, secretKey); + } + + /** + * 鑾峰彇绛惧悕 + */ + public static String getApiSingnature(JSONObject body, Long timeStamp, String secretKey) throws Exception { + String signature = ""; + try { + //鍙傛暟鎺掑簭锛屼繚璇佺鍚嶄竴鑷� + List fileds = new ArrayList<>(); + Set> set = body.entrySet(); + for (Entry entry : set) { + fileds.add(entry.getKey()); + } + //鍙傛暟鎵ц鎺掑簭 + Collections.sort(fileds); + Map params = new LinkedHashMap(); + for (String key : fileds) { + Object obj = body.get(key); + if(obj != null){ + params.put(key, obj); + } + } + String stringToSign = "timestamp="+timeStamp+"&body="+ params.toString(); + Mac mac = Mac.getInstance("HmacSHA256"); + mac.init(new SecretKeySpec(secretKey.getBytes("UTF-8"), "HmacSHA256")); + byte[] signatureBytes = mac.doFinal(stringToSign.getBytes("UTF-8")); + signature = new String(Base64.encodeBase64(signatureBytes)); + return signature; + } catch (Exception e) { + e.printStackTrace(); + throw new Exception("璁$畻绛惧悕閿欒"); + } + } + + public static JSONObject post(String url, JSONObject json) throws Exception { + return (JSONObject) post(url, json, new JSONObject()); + } + + + private static Object post(String url, JSONObject json, Object t) throws Exception { + HttpClient client = new DefaultHttpClient(); + HttpPost post = new HttpPost(url); + + post.setHeader("Content-Type", "application/json"); + String result = ""; + try { + StringEntity s = new StringEntity(json.toJSONString(), "utf-8"); + post.setEntity(s); + + // 鍙戦�佽姹� + HttpResponse httpResponse = client.execute(post); + + // 鑾峰彇鍝嶅簲杈撳叆娴� + InputStream inStream = httpResponse.getEntity().getContent(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inStream, "utf-8")); + StringBuilder strber = new StringBuilder(); + String line = null; + while ((line = reader.readLine()) != null){ + strber.append(line + "\n"); + } + inStream.close(); + result = strber.toString(); + if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { + return JSONObject.parseObject(result, t.getClass()); + } else { + return null; + } + } catch (Exception e) { + e.printStackTrace(); + throw new RuntimeException("璇锋眰寮傚父"); + } + } + + /** + * 鑾峰彇椤圭洰鏍囩 + * + * @return + */ + public static List listProjectLabel(String projectId) { + List labels = null; + try { + JSONObject body = new JSONObject(); + body.put("id", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/project/getProjectLabel", param); + if ((Integer)result.get("code") == 0) { + JSONArray jsonArray = result.getJSONArray("data"); + labels = jsonArray.toJavaList(ProjectLabel.class); + } + } catch (Exception e) { + e.printStackTrace(); + } + return labels; + } + + /** + * 鑾峰彇椤圭洰鍒楄〃 + * @return + */ + public static List listProject() { + List project = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("http://api.antbim.net/ant_api/project/getProjectList", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + project = jsonArray.toJavaList(Project.class); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return project; + } + + /** + * 鑾峰彇椤圭洰涓嬩换鍔℃竻鍗曞垪琛� + * + * @return + */ + public static JSONObject listInventory(String projectId) { + JSONObject result = new JSONObject(); + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + result = ApiUtil.post("https://api.antbim.net/ant_api/project/inventory/getInventoryList", param); + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + + + /** + * 鑾峰彇椤圭洰涓嬩换鍔$被鍨嬪垪琛� + * @return + */ + public static JSONObject listTaskType(String projectId) { + JSONObject result = new JSONObject(); + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + result = ApiUtil.post("https://api.antbim.net/ant_api/project/tasktype/getTasktypeList", param); + } catch (Exception e) { + e.printStackTrace(); + } + return result; + } + + /** + * 鑾峰彇椤圭洰鍐呬换鍔″垪琛� + * + * @return + */ + public static List listTaskByProject(String projectId, String startTime, String endTime) { + List tasks = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + body.put("projectId", projectId); + body.put("createtimeStart", startTime); + body.put("createtimeEnd", endTime); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByProject", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + tasks = jsonArray.toJavaList(Task.class); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + return tasks; + } + + /** + * 鑾峰彇浠诲姟璇︽儏 + * + * @return + */ + public static TaskDetail getTaskDetail(String taskId) { + TaskDetail detail = null; + try { + JSONObject body = new JSONObject(); + body.put("id", taskId); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskDetail", param); + if ((Integer)result.get("code") == 0) { + JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data"))); + detail = obj.toJavaObject(TaskDetail.class); + } + } catch (Exception e) { + e.printStackTrace(); + } + return detail; + } + + public static void main(String[] args) { + //List projects = ApiUtil.listProject(); + //System.out.println(ApiUtil.listTaskByProject("ecd77309ab1c4b83b54ee87b8ddd4f98", "2021-04-01", "2021-04-30")); + System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java new file mode 100644 index 000000000..e5ea5b6a9 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Project.java @@ -0,0 +1,92 @@ +package com.fangjia.script.statistics; + +/** + * @author shenjian + * @since 2021/5/8 + */ +public class Project { + private Integer taskCount; + private Integer taskendCount; + private String name; + private CreateUser createUser; + private String id; + private String state; + private String createDate; + + public Integer getTaskCount() { + return taskCount; + } + + public void setTaskCount(Integer taskCount) { + this.taskCount = taskCount; + } + + public Integer getTaskendCount() { + return taskendCount; + } + + public void setTaskendCount(Integer taskendCount) { + this.taskendCount = taskendCount; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public CreateUser getCreateUser() { + return createUser; + } + + public void setCreateUser(CreateUser createUser) { + this.createUser = createUser; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public String getCreateDate() { + return createDate; + } + + public void setCreateDate(String createDate) { + this.createDate = createDate; + } +} + +class CreateUser { + private String name; + private String userid; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public String getUserid() { + return userid; + } + + public void setUserid(String userid) { + this.userid = userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java new file mode 100644 index 000000000..bb5864998 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ProjectLabel.java @@ -0,0 +1,57 @@ +package com.fangjia.script.statistics; + +/** + * @author shenjian + * @since 2021/5/8 + */ +public class ProjectLabel { + + private String color; + private String name; + private Groups groups; + private String id; + + public String getColor() { + return color; + } + + public void setColor(String color) { + this.color = color; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public Groups getGroups() { + return groups; + } + + public void setGroups(Groups groups) { + this.groups = groups; + } + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } +} + +class Groups { + private String name; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java new file mode 100644 index 000000000..9a0c728c4 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java @@ -0,0 +1,64 @@ +package com.fangjia.script.statistics; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/8 + */ +@NoArgsConstructor +@Data +public class Task { + + @JSONField(name = "code") + private String code; + @JSONField(name = "response") + private ResponseDTO response; + @JSONField(name = "planBeginTime") + private String planBeginTime; + @JSONField(name = "name") + private String name; + @JSONField(name = "planEndTime") + private String planEndTime; + @JSONField(name = "id") + private String id; + @JSONField(name = "state") + private String state; + @JSONField(name = "inventory") + private InventoryDTO inventory; + @JSONField(name = "flow") + private FlowDTO flow; + @JSONField(name = "createDate") + private String createDate; + + @NoArgsConstructor + @Data + public static class ResponseDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } + + @NoArgsConstructor + @Data + public static class InventoryDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "id") + private String id; + @JSONField(name = "abbrev") + private String abbrev; + } + + @NoArgsConstructor + @Data + public static class FlowDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java new file mode 100644 index 000000000..3d0d4acfe --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskDetail.java @@ -0,0 +1,67 @@ +package com.fangjia.script.statistics; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/8 + */ +@NoArgsConstructor +@Data +public class TaskDetail { + + @JSONField(name = "code") + private String code; + @JSONField(name = "realityBeginDate") + private String realityBeginDate; + @JSONField(name = "assignDate") + private String assignDate; + @JSONField(name = "flowConten") + private String flowConten; + @JSONField(name = "description") + private String description; + @JSONField(name = "realityEndDate") + private String realityEndDate; + @JSONField(name = "workTime") + private String workTime; + @JSONField(name = "confirmDate") + private String confirmDate; + @JSONField(name = "response") + private ResponseDTO response; + @JSONField(name = "name") + private String name; + @JSONField(name = "inventoryId") + private String inventoryId; + @JSONField(name = "state") + private String state; + @JSONField(name = "id") + private String id; + @JSONField(name = "planEndDate") + private String planEndDate; + @JSONField(name = "planBeginDate") + private String planBeginDate; + @JSONField(name = "flow") + private FlowDTO flow; + @JSONField(name = "createDate") + private String createDate; + + @NoArgsConstructor + @Data + public static class ResponseDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } + + @NoArgsConstructor + @Data + public static class FlowDTO { + @JSONField(name = "name") + private String name; + @JSONField(name = "userid") + private String userid; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json new file mode 100644 index 000000000..10af777c0 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/label.json @@ -0,0 +1,38 @@ +{ + "code": 0, + "data": [ + { + "color": "74CBFF", + "name": "建行项目", + "groups": { + "name": "项目类别" + }, + "id": "3538604d04b1495585a45c23379b65f5" + }, + { + "color": "75ccff", + "name": "丁世菁", + "groups": { + "name": "商务负责人" + }, + "id": "c9055e8370464537a2eefc3aa925097b" + }, + { + "color": "88C896", + "name": "准备中", + "groups": { + "name": "项目进展" + }, + "id": "6138228645e347b3a1f6ed52ece457c5" + }, + { + "color": "75ccff", + "name": "董彦均", + "groups": { + "name": "商务负责人" + }, + "id": "8100d58e9a3e4c8cb9be9406147ee93b" + } + ], + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json new file mode 100644 index 000000000..f3e658cca --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/project.json @@ -0,0 +1,225 @@ +{ + "code": 0, + "data": { + "count": 18, + "list": [ + { + "taskCount": 2, + "taskendCount": 2, + "name": "建行配套数据采购项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "77478f60337840ccaf5ebb294ac1c987", + "state": "进行中", + "createDate": "2021-04-12 17:51:21" + }, + { + "taskCount": 3, + "taskendCount": 3, + "name": "房金部房产估值模型", + "createUser": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "id": "e7caf92da7be4c529c8733cbb196173e", + "state": "进行中", + "createDate": "2021-04-12 08:43:54" + }, + { + "taskCount": 1, + "taskendCount": 0, + "name": "地址标准化项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f7c307d9af794f4589f0a7027142e6e8", + "state": "进行中", + "createDate": "2021-04-01 10:03:56" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "企业经营地址库", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "59b1247428a445f5971f8f06091fb558", + "state": "进行中", + "createDate": "2021-03-22 15:10:41" + }, + { + "taskCount": 11, + "taskendCount": 8, + "name": "建信住房租金估价报告项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "ecd77309ab1c4b83b54ee87b8ddd4f98", + "state": "进行中", + "createDate": "2021-03-19 10:12:40" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "上数分地址标准化项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "4f22540647c74fc0a8277d798fbe9bfc", + "state": "进行中", + "createDate": "2021-03-18 16:50:19" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "建行开发商项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f82329c2390c4d2399f96ba21e92c950", + "state": "进行中", + "createDate": "2021-03-15 09:52:34" + }, + { + "taskCount": 9, + "taskendCount": 8, + "name": "房金部楼盘字典数据测试", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "457f11e89aad42b6a7bb289e8341ac04", + "state": "进行中", + "createDate": "2021-03-12 17:40:02" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "房价通", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "6a218757c86648559dacd9e4538d3f0a", + "state": "进行中", + "createDate": "2021-03-11 14:49:37" + }, + { + "taskCount": 1, + "taskendCount": 1, + "name": "亲邻科技", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "33c2a17a6d194168a2838c9ae4bd18b3", + "state": "进行中", + "createDate": "2021-03-10 11:46:36" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "福州农商银行", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "f1f7bf085c3b4033932b8e78d2a644ed", + "state": "进行中", + "createDate": "2021-03-10 10:43:23" + }, + { + "taskCount": 27, + "taskendCount": 25, + "name": "房价网基础数据建设", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "ddd31486abb645eaa8a48fb43d57045b", + "state": "进行中", + "createDate": "2021-03-10 10:27:53" + }, + { + "taskCount": 1, + "taskendCount": 1, + "name": "江南农村商业银行", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "8f18fddec76a4ca788946825a741abdc", + "state": "进行中", + "createDate": "2021-03-09 09:38:15" + }, + { + "taskCount": 72, + "taskendCount": 65, + "name": "建行楼盘数据治理项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "41bdf23cbf634035b694f92ba72b067f", + "state": "进行中", + "createDate": "2021-03-02 16:54:38" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "江苏农行项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "a10e63ac24b9487cb1dd1140dd2ffc94", + "state": "进行中", + "createDate": "2021-03-02 09:08:45" + }, + { + "taskCount": 0, + "taskendCount": 0, + "name": "浙商银行估值项目", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "53aa253f216542dfb8022dcbd18601cc", + "state": "进行中", + "createDate": "2021-03-02 09:07:54" + }, + { + "taskCount": 2, + "taskendCount": 2, + "name": "机房运维", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "fc76aa65155741c1a074305741a067eb", + "state": "进行中", + "createDate": "2021-03-02 09:01:42" + }, + { + "taskCount": 27, + "taskendCount": 26, + "name": "浙商银行本地化部署", + "createUser": { + "name": "王文艳", + "userid": "02245922356399" + }, + "id": "71e520aff8e243f3b04e9d6fea7867d1", + "state": "进行中", + "createDate": "2021-03-01 16:36:41" + } + ] + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json new file mode 100644 index 000000000..8026c9d69 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/task.json @@ -0,0 +1,85 @@ +{ + "code": 0, + "data": { + "count": 4, + "list": [ + { + "code": "5", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-06 09:00:00", + "name": "地图展示上海租金数据", + "planEndTime": "2021-04-06 18:00:00", + "id": "3bbb043ad0b44ea2a1213e725274e3ad", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-06 15:52:01" + }, + { + "code": "6", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-07 09:00:00", + "name": "租房报告预览下载;es迁移丢失数据同步", + "planEndTime": "2021-04-08 18:59:00", + "id": "8725262c17db408dab1f31473d50598f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-07 09:31:53" + }, + { + "code": "7", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-08 09:00:00", + "name": "根据新的租房报告进仁行接口修改", + "planEndTime": "2021-04-08 18:00:00", + "id": "309bdc4420e74868bda27884f4a46508", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-12 11:24:01" + }, + { + "code": "8", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-16 09:23:00", + "name": "估值页面报告优化及小学数据按标准导出", + "planEndTime": "2021-04-16 19:00:00", + "id": "45f9d101b18b48a08045de76cf2e53af", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-16 09:23:21" + } + ] + }, + "message": "成功" +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json new file mode 100644 index 000000000..362cb7398 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/taskDetail.json @@ -0,0 +1,29 @@ +{ + "code": 0, + "data": { + "code": "5", + "realityBeginDate": "2021-04-06 09:00:00", + "assignDate": "2021-04-06 15:52:01", + "flowConten": "1", + "description": "", + "realityEndDate": "2021-04-08 16:49:07", + "workTime": "1.00", + "confirmDate": "2021-04-08 16:49:07", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "name": "地图展示上海租金数据", + "inventoryId": "d00fcb9c5307499eab8f9798a23f6360", + "state": "已完成", + "id": "3bbb043ad0b44ea2a1213e725274e3ad", + "planEndDate": "2021-04-06 18:00:00", + "planBeginDate": "2021-04-06 09:00:00", + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-06 15:52:01" + }, + "message": "成功" +} From 17fa2bb110df6fc38ecf5fe65e7b203c29ca74d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 09:32:20 +0800 Subject: [PATCH 016/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/statistics/Export.groovy | 36 +++++++++++++++++-- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy index 7f53ec256..b8cb65980 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy @@ -1,6 +1,9 @@ import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils /** * 绩效相关信息导出 @@ -10,10 +13,13 @@ import com.fangjia.script.statistics.ProjectLabel */ class Export { + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-04-01" + static final String endTime = "2021-04-30" + static void main(String[] args) { + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] - result << ['统计时间', '2021年4月1日-2021年4月30日'] - result << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] List projects = ApiUtil.listProject(); for (Project project : projects) { @@ -31,10 +37,34 @@ class Export { ret[0] = label.getName() } } + // 获取项目总工时 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + Double totalTime = 0D; + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 先取绩效,不存在则取工时 + if (taskDetail.getFlowConten() != null) { + totalTime += Double.valueOf(taskDetail.getFlowConten()) + } else { + totalTime += Double.valueOf(taskDetail.getWorkTime()) + } + } + } + ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, totalTime) result << ret } + def sortedResult = [] + sortedResult << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] + for (def label : sort) { + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + } + } + } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "项目统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计"); } } From bf6af8637e8457a6797867695c8df8f1693b0c7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 11:48:10 +0800 Subject: [PATCH 017/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=A9=E6=95=88?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{Export.groovy => ProjectExport.groovy} | 6 +- .../shenjian/statistics/ScoreExport.groovy | 113 ++++++++++++++++++ 2 files changed, 116 insertions(+), 3 deletions(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/{Export.groovy => ProjectExport.groovy} (96%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy similarity index 96% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index b8cb65980..76ea84abe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/Export.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -6,12 +6,12 @@ import com.fangjia.script.statistics.TaskDetail import org.apache.commons.collections.CollectionUtils /** - * 绩效相关信息导出 + * 项目导出 * * @author shenjian * @since 2021/5/8 */ -class Export { +class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01" @@ -64,7 +64,7 @@ class Export { } } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计"); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy new file mode 100644 index 000000000..baead79bc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -0,0 +1,113 @@ +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils + +/** + * 绩效导出 + * + * @author shenjian + * @since 2021/5/10 + */ +class ScoreExport { + + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-04-01" + static final String endTime = "2021-04-30" + + static void main(String[] args) { + def sortedResult = [] + sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def result = [] + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[2] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[3] == null) { + ret[3] = label.getName() + } else { + ret[3] = ret[3] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + // 获取项目下任务 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + Double totalTime = 0D; + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true + } + } + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + } + } + } + result << ret + } + def allTotal = [] + for (def label : sort) { + def total = [] + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + for (int i = 4; i < ret.size(); i++) { + if (total[i] == null) { + total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i]) + } else { + total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i])) + } + } + } + } + + total[1] = 0 + total[2] = "小计" + sortedResult << total + for (int i = 4; i < total.size(); i++) { + if (allTotal[i] == null) { + allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i]) + } else { + allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + } + allTotal[2] = "人员绩效总计" + sortedResult << allTotal + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + } +} + + + + + + + + + + + + From ab7213dd148ffddb87c80544f5a587ed89434cf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 14:36:21 +0800 Subject: [PATCH 018/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/TaskExport.groovy | 113 +++ .../fangjia/script/statistics/ApiUtil.java | 39 +- .../script/statistics/json/personTask.json | 928 ++++++++++++++++++ 3 files changed, 1079 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy new file mode 100644 index 000000000..f2e64dd9e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -0,0 +1,113 @@ +import com.fangjia.script.statistics.ApiUtil +import com.fangjia.script.statistics.Project +import com.fangjia.script.statistics.ProjectLabel +import com.fangjia.script.statistics.Task +import com.fangjia.script.statistics.TaskDetail +import org.apache.commons.collections.CollectionUtils + +/** + * 任务统计 + * + * @author shenjian + * @since 2021/5/10 + */ +class TaskExport { + + static final String time = "2021年4月1日-2021年4月30日"; + static final String startTime = "2021-04-01" + static final String endTime = "2021-04-30" + + static void main(String[] args) { + def sortedResult = [] + sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def result = [] + List projects = ApiUtil.listProject(); + for (Project project : projects) { + def ret = [] + List labels = ApiUtil.listProjectLabel(project.getId()) + ret[2] = project.getName() + for (ProjectLabel label : labels) { + if ("商务负责人" == label.getGroups().getName()) { + if (ret[3] == null) { + ret[3] = label.getName() + } else { + ret[3] = ret[3] + "、" + label.getName() + } + } else if ("项目类别" == label.getGroups().getName()) { + ret[0] = label.getName() + } + } + // 获取项目下任务 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + Double totalTime = 0D; + if (CollectionUtils.isNotEmpty(tasks)) { + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true + } + } + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + } + } + } + result << ret + } + def allTotal = [] + for (def label : sort) { + def total = [] + for (def ret : result) { + if (ret[0] == label) { + sortedResult << ret + for (int i = 4; i < ret.size(); i++) { + if (total[i] == null) { + total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i]) + } else { + total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i])) + } + } + } + } + + total[1] = 0 + total[2] = "小计" + sortedResult << total + for (int i = 4; i < total.size(); i++) { + if (allTotal[i] == null) { + allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i]) + } else { + allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) + } + } + allTotal[2] = "人员绩效总计" + sortedResult << allTotal + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + } +} + + + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 1c2836659..33c96ae42 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -267,6 +267,42 @@ public class ApiUtil { return tasks; } + /** + * 鑾峰彇浜哄憳浠诲姟鍒楄〃 + */ + public static List listTaskByPerson(String startTime, String endTime) { + List tasks = null; + try { + JSONObject body = new JSONObject(); + body.put("limitStart", "0"); + body.put("limitNum", "50"); + //body.put("userid", ""); + body.put("type", ""); + body.put("timezone", ""); + body.put("createtimeStart", startTime); + body.put("createtimeEnd", endTime); + + long timeStamp = new Date().getTime(); + + String signature = ApiUtil.getApiSingnature(body.toJSONString(), timeStamp, secretKey); + JSONObject param = new JSONObject(); + param.put("signature", signature); + param.put("timeStamp", timeStamp); + param.put("corpid", corpId); + param.put("body", body); + JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByPerson", param); + if ((Integer)result.get("code") == 0) { + if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { + JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); + tasks = jsonArray.toJavaList(Task.class); + } + } + } catch (Exception ex) { + ex.printStackTrace(); + } + return tasks; + } + /** * 鑾峰彇浠诲姟璇︽儏 * @@ -300,6 +336,7 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); //System.out.println(ApiUtil.listTaskByProject("ecd77309ab1c4b83b54ee87b8ddd4f98", "2021-04-01", "2021-04-30")); - System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + System.out.println(ApiUtil.listTaskByPerson("2021-04-01", "2021-04-30")); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json new file mode 100644 index 000000000..d7510e331 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/personTask.json @@ -0,0 +1,928 @@ +{ + "code": 0, + "data": { + "count": 176, + "list": [ + { + "code": "78", + "response": { + "name": "黄春英", + "userid": "143809622239903024" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "小区别名校验", + "planEndTime": "2021-05-14 23:59:00", + "id": "5c8f51d3971241eea4b1efc92b20fe5c", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 10:53:51" + }, + { + "code": "77", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-05-15 23:59:00", + "id": "3e12b0053aca4334ad28cd9bc30679f0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 10:19:16" + }, + { + "code": "76", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-05-15 23:59:00", + "id": "d6ccafed4b284951a5848ba62cfc00f0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:23:12" + }, + { + "code": "75", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "建行表整理及系统测试", + "planEndTime": "2021-05-15 23:59:00", + "id": "506a33123d984df98a05973fcc76f3d0", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:12:20" + }, + { + "code": "74", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-05-10 00:00:00", + "name": "系统测试", + "planEndTime": "2021-05-15 23:59:00", + "id": "bb3f9d174f9341b58b89203b3b326b9f", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 09:11:22" + }, + { + "code": "73", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-05-10 08:45:00", + "name": "导出小区别名数据给到人工检验,并整理人工反馈的小区别名校验结果;\n数据治理模型上线相关工作;", + "planEndTime": "2021-05-15 23:59:00", + "id": "ca1bae5186164b65b0807b597634317e", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-10 08:32:43" + }, + { + "code": "1.1", + "response": { + "name": "王辰岑", + "userid": "02245922215748" + }, + "planBeginTime": "2021-05-08 09:30:00", + "name": "网站注册及申请入驻供应商", + "planEndTime": "2021-05-08 18:00:00", + "id": "b1753f50c7a149f3a85c3a2fa0f180a8", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "7f01818d37c84c4d9337ef288623f5eb", + "abbrev": "A" + }, + "createDate": "2021-05-08 17:52:10" + }, + { + "code": "1", + "response": { + "name": "王辰岑", + "userid": "02245922215748" + }, + "name": "平台申请", + "id": "d357543551d14b22a91d2bd93b7afd31", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "7f01818d37c84c4d9337ef288623f5eb", + "abbrev": "A" + }, + "flow": { + "name": "蒋晓春", + "userid": "02245922395062" + }, + "createDate": "2021-05-08 17:49:05" + }, + { + "code": "11", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-05-08 09:00:00", + "name": "租金估值权限逻辑修改", + "planEndTime": "2021-05-08 19:00:00", + "id": "d0f6f2584e7440739c5b603f942c2bb7", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-05-08 09:42:10" + }, + { + "code": "27", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-06 09:00:00", + "name": "ES机器学习三台节点搭建", + "planEndTime": "2021-04-06 18:59:00", + "id": "a725c9be2bb2446eb180129fb58094d5", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:57:54" + }, + { + "code": "72", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "建行系统测试", + "planEndTime": "2021-05-08 23:59:00", + "id": "0c938c9b36054c84bc69ba166b31dea2", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:55:35" + }, + { + "code": "71", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "数据融合人工反馈处理", + "planEndTime": "2021-05-08 23:59:00", + "id": "daa8d8e8a69842eaae621a33e8690e55", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 15:43:55" + }, + { + "code": "26", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-05-07 09:00:00", + "name": "分析爬虫抓取情况,导出亲邻数据,导出四川锦城数据", + "planEndTime": "2021-05-07 18:00:00", + "id": "1c9085be784542fb917a2ff2767388c9", + "state": "待确认", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-05-07 15:39:42" + }, + { + "code": "70", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-07 00:00:00", + "name": "新增安居客小区详情解析,优化安居客小区详情抓取", + "id": "e7717e32c9864511aabfedd41117d6ca", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-07 14:37:30" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "微元素帐号整理", + "id": "4de48c77988743a5ab83343b40adee97", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "907da15cb4a54184bff9b32345300d2a", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:54:24" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 14:00:00", + "name": "公司名称及地址匹配", + "id": "286ad20909ca4b27bd70c34eeea6a3c1", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "270452cd5725475095c7e668ea137481", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:22:42" + }, + { + "code": "69", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-05-06 09:30:00", + "name": "新增搜房小区列表抓取及解析\n导出新租金米数据", + "planEndTime": "2021-05-06 14:00:00", + "id": "35df605842d941a58b94d6b5e3c8e5b7", + "state": "进行中", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 14:21:21" + }, + { + "code": "68", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-05-08 23:59:00", + "id": "e3758204453b4858a02c33f7f0ceaf45", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 09:41:20" + }, + { + "code": "67", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-05-06 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-05-08 23:59:00", + "id": "613d42946b694b24b9c6d29a7fb3cbc8", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 09:41:00" + }, + { + "code": "66", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-05-06 08:45:00", + "name": "1.将小区别名数据导出给人工校验2.将78数据库中的人工标注数据,经纬度围栏数据放到85、88数据库", + "planEndTime": "2021-05-08 23:59:00", + "id": "2410d7840bff48af8b2202c7c7271cfb", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-05-06 08:30:36" + }, + { + "code": "10", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-05 00:00:00", + "name": "优化诸葛找房抓取稠租房少的问题\n新增搜房通过入小区页面抓取租房户房源\n及验证曹老师说抓取少的网站重抓", + "planEndTime": "2021-04-07 23:59:00", + "id": "c37324ae475149cfb0648731934873a9", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:35:09" + }, + { + "code": "25", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-29 00:00:00", + "name": "网站种子配置及福州二手房抓取", + "planEndTime": "2021-04-30 23:59:00", + "id": "a1326b626acf444193969c755287de1c", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:32:07" + }, + { + "code": "9", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-03 00:00:00", + "name": "新增我爱我家,百姓,赶集的抓取解析,\n新增我爱我家成交抓取,\n修改诸葛找房抓取,解析", + "planEndTime": "2021-04-03 23:59:00", + "id": "73e49abc26574e78b696113bcf200019", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "d00fcb9c5307499eab8f9798a23f6360", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:10:56" + }, + { + "code": "65", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "建行数据融合统计结果(彦均)", + "planEndTime": "2021-04-25 23:59:00", + "id": "be85282dc57d46e3a36fc0801107e605", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:03:21" + }, + { + "code": "3", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "建行数据导出(彦均)\n房源量小于20的小区\n小区统计", + "planEndTime": "2021-04-19 23:59:00", + "id": "af3c862c7c674f97855061cb923fbfef", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "ac8c4d70ea3842c2a6c628a64628989a", + "abbrev": "A" + }, + "createDate": "2021-04-30 09:00:56" + }, + { + "code": "24", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-29 00:00:00", + "name": "通过小区抓安居客房源,配完,反爬厉害\n分析抓取现状\n总结输出房源方案", + "planEndTime": "2021-04-29 23:59:00", + "id": "bfccb3c436af4d1a9263ae4879dc7fee", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-29 18:29:05" + }, + { + "code": "1", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "企址库抓取\n尝试国家信息,天眼查,爱企查有js问题\n个企查查,启信宝有豋录问题,88不全,最后抓顺企", + "planEndTime": "2021-04-22 23:59:00", + "id": "b495c086d3fa4a6d8e6d7319fdef303f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "270452cd5725475095c7e668ea137481", + "abbrev": "A" + }, + "createDate": "2021-04-29 12:05:12" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-26 00:00:00", + "name": "建行案例测试批量评估价格,并并导出房源与神经网络j小区", + "planEndTime": "2021-04-28 23:59:00", + "id": "68afd0578fab469d8371fb510c1f8af9", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "ac8c4d70ea3842c2a6c628a64628989a", + "abbrev": "A" + }, + "createDate": "2021-04-29 11:51:14" + }, + { + "code": "26", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-26 00:00:00", + "name": "浙商本地化部署项目验证环境发布", + "planEndTime": "2021-04-28 23:59:00", + "id": "91ea49c8f5e747b18660dec02da7bfeb", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "b4abc1f7cad142c4be5e5b301ae08190", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-29 09:44:24" + }, + { + "code": "25", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-26 09:00:00", + "name": "浙商本地化验证环境部署工作", + "planEndTime": "2021-04-28 17:59:00", + "id": "09c28bed9a43475c9bd24a69066170d4", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "b4abc1f7cad142c4be5e5b301ae08190", + "abbrev": "A" + }, + "createDate": "2021-04-26 11:34:36" + }, + { + "code": "23", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "Python工具包封装", + "planEndTime": "2021-04-25 23:59:00", + "id": "f3a00c2e64184796a4a12ab37ce60ac5", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-25 18:24:34" + }, + { + "code": "64", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "系统测试,数据统计", + "planEndTime": "2021-04-30 23:59:00", + "id": "8b8be8a4a3424ac9a3cd74145d02f26f", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 09:27:25" + }, + { + "code": "63", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-04-30 23:59:00", + "id": "1140039f97da40639763101258f4c300", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 09:14:40" + }, + { + "code": "62", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-30 23:59:00", + "id": "9c2064b98c814b3183189c6d6a68a068", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:41:04" + }, + { + "code": "61", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "96f4d6e9851b44fc84eb20a8fd58504a", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:39:14" + }, + { + "code": "60", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-04-25 08:45:00", + "name": "1.在82.83.84服务器上运行测试代码;\n2.编写”数据质量监测维度信息统计”的代码", + "planEndTime": "2021-04-30 23:59:00", + "id": "a354e6df53e34a9581fa5988f4334dfa", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-25 08:28:25" + }, + { + "code": "22", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-04-25 00:00:00", + "name": "租金模型开发", + "planEndTime": "2021-04-27 23:59:00", + "id": "3f569aa260b04da88224834d56f43427", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-25 07:58:48" + }, + { + "code": "21", + "response": { + "name": "沈健", + "userid": "010828030128881885" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "pyspider新增获取小区id接口及项目爬取时间监控", + "planEndTime": "2021-04-22 23:59:00", + "id": "f8122a43929646e8acfdbbf70091bf3c", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-21 09:44:31" + }, + { + "code": "59", + "response": { + "name": "黄春英", + "userid": "143809622239903024" + }, + "planBeginTime": "2021-04-20 00:00:00", + "name": "别名表校验", + "planEndTime": "2021-04-25 23:59:00", + "id": "6a26dbc7534240f88861873177bca3ee", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-20 09:02:45" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "深圳房价角通室厅必填\n从室号解析楼层", + "planEndTime": "2021-04-19 18:00:00", + "id": "b4001b1b33fe486d93897d737dd950a3", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "cc6b1205b3d547f5867f44ddf5d568df", + "abbrev": "A" + }, + "createDate": "2021-04-19 19:23:25" + }, + { + "code": "2", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-13 00:00:00", + "name": "建行学校抓取合肥,苏苏州,西安", + "planEndTime": "2021-04-16 23:59:00", + "id": "e26d64fb288a4558bec894035db70f27", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "0fbd26a5a4914d92802352502819abd8", + "abbrev": "A" + }, + "createDate": "2021-04-19 19:14:47" + }, + { + "code": "58", + "response": { + "name": "徐鹤", + "userid": "0145075527798932" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "建行数据系统开发", + "planEndTime": "2021-04-23 23:59:00", + "id": "de19c283d30843c9971d1eb5ba4aa157", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 16:14:04" + }, + { + "code": "20", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "修改均价测试系统bug", + "planEndTime": "2021-04-19 18:00:00", + "id": "d6f6e37425d24b2ba04ef1ce64958358", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-19 13:29:35" + }, + { + "code": "57", + "response": { + "name": "王思璐", + "userid": "044615161129218014" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "数据清洗", + "planEndTime": "2021-04-24 23:59:00", + "id": "a5d3e59e2fae4f42bc5097d50096edab", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:19:07" + }, + { + "code": "56", + "response": { + "name": "许镕杰", + "userid": "014163606524721483" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "融合后别名表数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "8ef405124f054f578fa4b21c7863c415", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:18:50" + }, + { + "code": "55", + "response": { + "name": "李亿亿", + "userid": "01052556214326059694" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "小区别名数据校验", + "planEndTime": "2021-04-23 23:59:00", + "id": "51d0a97ac69b48c89926ee7f2736e62d", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 09:06:04" + }, + { + "code": "54", + "response": { + "name": "穆良书", + "userid": "055551305431136413" + }, + "planBeginTime": "2021-04-19 00:00:00", + "name": "楼盘治理上线联调", + "planEndTime": "2021-04-23 23:59:00", + "id": "d08c7c2c852048aebc8aa59d48cfbd21", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 08:43:11" + }, + { + "code": "53", + "response": { + "name": "徐翔翔", + "userid": "083765536424557072" + }, + "planBeginTime": "2021-04-19 08:45:00", + "name": "1.根据人工融合校验,优化清洗关键字和规则;2.在Linux的82.83.84服务器上测试代码", + "planEndTime": "2021-04-23 23:59:00", + "id": "2af0fe9397094718941cd081e5dfbe53", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "3ef84a3a16b346e3a077139362cba07a", + "abbrev": "A" + }, + "createDate": "2021-04-19 08:26:10" + }, + { + "code": "19", + "response": { + "name": "高娟", + "userid": "02245922257828" + }, + "planBeginTime": "2021-04-19 09:00:00", + "name": "友达二手成交数据导入", + "planEndTime": "2021-04-19 18:00:00", + "id": "3a1891dfc95c4d39ac33ce64761a4a57", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "createDate": "2021-04-19 00:04:47" + }, + { + "code": "18", + "response": { + "name": "王文艳", + "userid": "02245922356399" + }, + "planBeginTime": "2021-04-16 00:00:00", + "name": "深圳指导价入库与神经网络价格修复", + "planEndTime": "2021-04-16 23:59:00", + "id": "4999543a15ff429a85875348df048726", + "state": "已完成", + "inventory": { + "name": "默认清单", + "id": "39870777580f450185c7357e57cc110c", + "abbrev": "A" + }, + "flow": { + "name": "王文艳", + "userid": "02245922356399" + }, + "createDate": "2021-04-16 18:59:31" + } + ] + }, + "message": "成功" +} From 88211095fdc0167ad7c6af9cf5090e3197a6eeb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 16:16:53 +0800 Subject: [PATCH 019/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=9F=8E=E5=B8=82?= =?UTF-8?q?=E6=95=B0=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/crawl/City58.groovy | 277 ++++++++++++++++++ 1 file changed, 277 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy new file mode 100644 index 000000000..78f033fd4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy @@ -0,0 +1,277 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 58城市简写 + * + * @author shenjian + * @since 2021/5/10 + */ +class City58 { + + static String cityList = "{\n" + + " \"安徽\":{\n" + + " \"合肥\":\"hf|837\",\"芜湖\":\"wuhu|2045\",\"蚌埠\":\"bengbu|3470\",\"阜阳\":\"fy|2325\",\"淮南\":\"hn|2319\",\n" + + " \"安庆\":\"anqing|3251\",\"宿州\":\"suzhou|3359\",\"六安\":\"la|2328\",\"淮北\":\"huaibei|9357\",\"滁州\":\"chuzhou|10266\",\n" + + " \"马鞍山\":\"mas|2039\",\"铜陵\":\"tongling|10285\",\"宣城\":\"xuancheng|5633\",\"亳州\":\"bozhou|2329\",\"黄山\":\"huangshan|2323\",\n" + + " \"池州\":\"chizhou|10260\",\"巢湖\":\"ch|10229\",\"和县\":\"hexian|10892\",\"霍邱\":\"hq|11226\",\"桐城\":\"tongcheng|11296\",\n" + + " \"宁国\":\"ningguo|5645\",\"天长\":\"tianchang|10273\",\"东至\":\"dongzhi|10262\",\"无为\":\"wuweixian|10232\"\n" + + " }\n" + + " ,\n" + + " \"福建\":{\n" + + " \"福州\":\"fz|304\",\"厦门\":\"xm|606\",\"泉州\":\"qz|291\",\"莆田\":\"pt|2429\",\"漳州\":\"zhangzhou|710\",\n" + + " \"宁德\":\"nd|7951\",\"三明\":\"sm|2048\",\"南平\":\"np|10291\",\"龙岩\":\"ly|6752\",\"武夷山\":\"wuyishan|10761\",\n" + + " \"石狮\":\"shishi|296\",\"晋江\":\"jinjiangshi|297\",\"南安\":\"nananshi|293\",\"龙海\":\"longhai|713\",\"上杭\":\"shanghangxian|6757\",\n" + + " \"福安\":\"fuanshi|7969\",\"福鼎\":\"fudingshi|7970\",\"安溪\":\"anxixian|7100\",\"永春\":\"yongchunxian|7101\",\"永安\":\"yongan|2133\",\n" + + " \"漳浦\":\"zhangpu|717\"\n" + + " }\n" + + " ,\n" + + " \"广东\":{\n" + + " \"深圳\":\"sz|4\",\"广州\":\"gz|3\",\"东莞\":\"dg|413\",\"佛山\":\"fs|222\",\"中山\":\"zs|771\",\n" + + " \"珠海\":\"zh|910\",\"惠州\":\"huizhou|722\",\"江门\":\"jm|629\",\"汕头\":\"st|783\",\"湛江\":\"zhanjiang|791\",\n" + + " \"肇庆\":\"zq|901\",\"茂名\":\"mm|679\",\"揭阳\":\"jy|927\",\"梅州\":\"mz|9389\",\"清远\":\"qingyuan|7303\",\n" + + " \"阳江\":\"yj|2284\",\"韶关\":\"sg|2192\",\"河源\":\"heyuan|10467\",\"云浮\":\"yf|10485\",\"汕尾\":\"sw|9449\",\n" + + " \"潮州\":\"chaozhou|10461\",\"台山\":\"taishan|11263\",\"阳春\":\"yangchun|8566\",\"顺德\":\"sd|8716\",\"惠东\":\"huidong|725\",\n" + + " \"博罗\":\"boluo|726\",\"海丰\":\"haifengxian|9444\",\"开平\":\"kaipingshi|634\",\"陆丰\":\"lufengshi|9456\"\n" + + " }\n" + + " ,\n" + + " \"广西\":{\n" + + " \"南宁\":\"nn|845\",\"柳州\":\"liuzhou|7133\",\"桂林\":\"gl|1039\",\"玉林\":\"yulin|2337\",\"梧州\":\"wuzhou|2046\",\n" + + " \"北海\":\"bh|10536\",\"贵港\":\"gg|6770\",\"钦州\":\"qinzhou|2335\",\"百色\":\"baise|10513\",\"河池\":\"hc|2340\",\n" + + " \"来宾\":\"lb|10552\",\"贺州\":\"hezhou|10549\",\"防城港\":\"fcg|10539\",\"崇左\":\"chongzuo|10524\",\"桂平\":\"guipingqu|6774\",\n" + + " \"北流\":\"beiliushi|9168\",\"博白\":\"bobaixian|9173\",\"岑溪\":\"cenxi|2119\"\n" + + " }\n" + + " ,\n" + + " \"贵州\":{\n" + + " \"贵阳\":\"gy|2015\",\"遵义\":\"zunyi|7620\",\"黔东南\":\"qdn|9363\",\"黔南\":\"qn|10492\",\"六盘水\":\"lps|10506\",\n" + + " \"毕节\":\"bijie|10564\",\"铜仁\":\"tr|10417\",\"安顺\":\"anshun|7468\",\"黔西南\":\"qxn|10434\",\"仁怀\":\"renhuaishi|7628\",\n" + + " \"清镇\":\"qingzhen|12703\"\n" + + " }\n" + + " ,\n" + + " \"甘肃\":{\n" + + " \"兰州\":\"lz|952\",\"天水\":\"tianshui|8601\",\"白银\":\"by|10304\",\"庆阳\":\"qingyang|10475\",\"平凉\":\"pl|7154\",\n" + + " \"酒泉\":\"jq|10387\",\"张掖\":\"zhangye|10454\",\"武威\":\"wuwei|10448\",\"定西\":\"dx|10322\",\"金昌\":\"jinchang|7428\",\n" + + " \"陇南\":\"ln|10415\",\"临夏\":\"linxia|7112\",\"嘉峪关\":\"jyg|10362\",\"甘南\":\"gn|10343\",\"敦煌\":\"dunhuang|10390\"\n" + + " }\n" + + " ,\n" + + " \"海南\":{\n" + + " \"海口\":\"haikou|2053\",\"三亚\":\"sanya|2422\",\"五指山\":\"wzs|9952\",\"三沙\":\"sansha|13722\",\"琼海\":\"qh|10136\",\n" + + " \"文昌\":\"wenchang|9984\",\"万宁\":\"wanning|10022\",\"屯昌\":\"tunchang|10044\",\"琼中\":\"qiongzhong|10064\",\"陵水\":\"lingshui|10184\",\n" + + " \"东方\":\"df|10250\",\"定安\":\"da|10303\",\"澄迈\":\"cm|10331\",\"保亭\":\"baoting|10367\",\"白沙\":\"baish|10380\",\n" + + " \"儋州\":\"danzhou|10394\"\n" + + " }\n" + + " ,\n" + + " \"河南\":{\n" + + " \"郑州\":\"zz|342\",\"洛阳\":\"luoyang|556\",\"新乡\":\"xx|1016\",\"南阳\":\"ny|592\",\"许昌\":\"xc|977\",\n" + + " \"平顶山\":\"pds|1005\",\"安阳\":\"ay|1096\",\"焦作\":\"jiaozuo|3266\",\"商丘\":\"sq|1029\",\"开封\":\"kaifeng|2342\",\n" + + " \"濮阳\":\"puyang|2346\",\"周口\":\"zk|933\",\"信阳\":\"xy|8694\",\"驻马店\":\"zmd|1067\",\"漯河\":\"luohe|2347\",\n" + + " \"三门峡\":\"smx|9317\",\"鹤壁\":\"hb|2344\",\"济源\":\"jiyuan|9918\",\"明港\":\"mg|8541\",\"鄢陵\":\"yanling|9123\",\n" + + " \"禹州\":\"yuzhou|979\",\"长葛\":\"changge|9344\",\"灵宝\":\"lingbaoshi|9307\",\"杞县\":\"qixianqu|7389\",\"汝州\":\"ruzhou|1010\",\n" + + " \"项城\":\"xiangchengshi|935\",\"偃师\":\"yanshiqu|7121\",\"长垣\":\"changyuan|5936\",\"滑县\":\"huaxian|5405\",\"林州\":\"linzhou|1101\",\n" + + " \"沁阳\":\"qinyang|3268\",\"孟州\":\"mengzhou|3267\",\"温县\":\"wenxian|7312\",\"尉氏\":\"weishixian|7391\",\"兰考\":\"lankaoxian|7393\",\n" + + " \"通许\":\"tongxuxian|7390\",\"新安\":\"lyxinan|11217\",\"伊川\":\"yichuan|11220\",\"孟津\":\"mengjinqu|7122\",\"宜阳\":\"lyyiyang|11219\",\n" + + " \"舞钢\":\"wugang|1011\",\"永城\":\"yongcheng|1032\",\"睢县\":\"suixian|1038\",\"鹿邑\":\"luyi|939\",\"渑池\":\"yingchixian|9322\",\n" + + " \"沈丘\":\"shenqiu|942\",\"太康\":\"taikang|938\",\"商水\":\"shangshui|936\",\"淇县\":\"qixianq|9186\",\"浚县\":\"junxian|9185\",\n" + + " \"范县\":\"fanxian|7285\",\"固始\":\"gushixian|8698\",\"淮滨\":\"huaibinxian|8702\",\"邓州\":\"dengzhou|595\",\"新野\":\"xinye|603\"\n" + + " }\n" + + " ,\n" + + " \"黑龙江\":{\n" + + " \"哈尔滨\":\"hrb|202\",\"大庆\":\"dq|375\",\"齐齐哈尔\":\"qqhr|5853\",\"牡丹江\":\"mdj|3489\",\"绥化\":\"suihua|6718\",\n" + + " \"佳木斯\":\"jms|6776\",\"鸡西\":\"jixi|7289\",\"双鸭山\":\"sys|9837\",\"鹤岗\":\"hegang|9061\",\"黑河\":\"heihe|9862\",\n" + + " \"伊春\":\"yich|9773\",\"七台河\":\"qth|9848\",\"大兴安岭\":\"dxal|9878\",\"安达\":\"shanda|6720\",\"肇东\":\"shzhaodong|6721\",\n" + + " \"肇州\":\"zhaozhou|382\"\n" + + " }\n" + + " ,\n" + + " \"湖北\":{\n" + + " \"武汉\":\"wh|158\",\"宜昌\":\"yc|858\",\"襄阳\":\"xf|891\",\"荆州\":\"jingzhou|3479\",\"十堰\":\"shiyan|2032\",\n" + + " \"黄石\":\"hshi|1734\",\"孝感\":\"xiaogan|3434\",\"黄冈\":\"hg|2299\",\"恩施\":\"es|2302\",\"荆门\":\"jingmen|2296\",\n" + + " \"咸宁\":\"xianning|9617\",\"鄂州\":\"ez|9709\",\"随州\":\"suizhou|9656\",\"潜江\":\"qianjiang|9669\",\"天门\":\"tm|9517\",\n" + + " \"仙桃\":\"xiantao|9736\",\"神农架\":\"snj|9605\",\"宜都\":\"yidou|864\",\"汉川\":\"hanchuan|3439\",\"枣阳\":\"zaoyang|896\",\n" + + " \"武穴\":\"wuxueshi|7362\",\"钟祥\":\"zhongxiangshi|9119\",\"京山\":\"jingshanxian|9117\",\"沙洋\":\"shayangxian|9118\",\"松滋\":\"songzi|3484\",\n" + + " \"广水\":\"guangshuishi|9657\",\"赤壁\":\"chibishi|9623\",\"老河口\":\"laohekou|895\",\"谷城\":\"gucheng|899\",\"宜城\":\"yichengshi|897\",\n" + + " \"南漳\":\"nanzhang|898\",\"云梦\":\"yunmeng|3438\",\"安陆\":\"anlu|3442\",\"大悟\":\"dawu|3437\",\"孝昌\":\"xiaochang|3436\",\n" + + " \"当阳\":\"dangyang|865\",\"枝江\":\"zhijiang|866\",\"嘉鱼\":\"jiayuxian|9624\",\"随县\":\"suixia|9660\"\n" + + " }\n" + + " ,\n" + + " \"湖南\":{\n" + + " \"长沙\":\"cs|414\",\"株洲\":\"zhuzhou|1086\",\"益阳\":\"yiyang|10198\",\"常德\":\"changde|872\",\"衡阳\":\"hy|914\",\n" + + " \"湘潭\":\"xiangtan|2047\",\"岳阳\":\"yy|821\",\"郴州\":\"chenzhou|5695\",\"邵阳\":\"shaoyang|2303\",\"怀化\":\"hh|5756\",\n" + + " \"永州\":\"yongzhou|2307\",\"娄底\":\"ld|9481\",\"湘西\":\"xiangxi|10219\",\"张家界\":\"zjj|6788\",\"醴陵\":\"liling|1091\",\n" + + " \"澧县\":\"lixian|876\",\"桂阳\":\"czguiyang|5699\",\"资兴\":\"zixing|5698\",\"永兴\":\"yongxing|5701\",\"常宁\":\"changningshi|921\",\n" + + " \"祁东\":\"qidongxian|5690\",\"衡东\":\"hengdong|5693\",\"冷水江\":\"lengshuijiangshi|9470\",\"涟源\":\"lianyuanshi|9471\",\"双峰\":\"shuangfengxian|9473\",\n" + + " \"邵阳县\":\"shaoyangxian|6955\",\"邵东\":\"shaodongxian|6954\",\"沅江\":\"yuanjiangs|10201\",\"南县\":\"nanxian|10202\",\"祁阳\":\"qiyang|8532\",\n" + + " \"湘阴\":\"xiangyin|828\",\"华容\":\"huarong|830\",\"慈利\":\"cilixian|6791\",\"攸县\":\"zzyouxian|1095\"\n" + + " }\n" + + " ,\n" + + " \"河北\":{\n" + + " \"石家庄\":\"sjz|241\",\"保定\":\"bd|424\",\"唐山\":\"ts|276\",\"廊坊\":\"lf|772\",\"邯郸\":\"hd|572\",\n" + + " \"秦皇岛\":\"qhd|1078\",\"沧州\":\"cangzhou|652\",\"邢台\":\"xt|751\",\"衡水\":\"hs|993\",\"张家口\":\"zjk|3328\",\n" + + " \"承德\":\"chengde|6760\",\"定州\":\"dingzhou|8398\",\"馆陶\":\"gt|8706\",\"张北\":\"zhangbei|11201\",\"赵县\":\"zx|9048\",\n" + + " \"正定\":\"zd|3198\",\"迁安市\":\"qianan|284\",\"任丘\":\"renqiu|656\",\"三河\":\"sanhe|776\",\"武安\":\"wuan|577\",\n" + + " \"雄安新区\":\"xionganxinqu|111234\",\"燕郊\":\"lfyanjiao|12804\",\"涿州\":\"zhuozhou|428\",\"河间\":\"hejian|658\",\"黄骅\":\"huanghua|657\",\n" + + " \"沧县\":\"cangxian|659\",\"磁县\":\"cixian|591\",\"涉县\":\"shexian|14059\",\"霸州\":\"bazhou|775\",\"香河\":\"xianghe|5395\",\n" + + " \"固安\":\"lfguan|12803\",\"遵化市\":\"zunhua|283\",\"迁西\":\"qianxixian|7061\",\"玉田\":\"yutianxian|7060\",\"滦南\":\"luannanxian|7066\",\n" + + " \"沙河\":\"shaheshi|755\"\n" + + " }\n" + + " ,\n" + + " \"江苏\":{\n" + + " \"苏州\":\"su|5\",\"南京\":\"nj|172\",\"无锡\":\"wx|93\",\"常州\":\"cz|463\",\"徐州\":\"xz|471\",\n" + + " \"南通\":\"nt|394\",\"扬州\":\"yz|637\",\"盐城\":\"yancheng|613\",\"淮安\":\"ha|968\",\"连云港\":\"lyg|2049\",\n" + + " \"泰州\":\"taizhou|693\",\"宿迁\":\"suqian|2350\",\"镇江\":\"zj|645\",\"沭阳\":\"shuyang|5772\",\"大丰\":\"dafeng|11279\",\n" + + " \"如皋\":\"rugao|397\",\"启东\":\"qidong|400\",\"溧阳\":\"liyang|469\",\"海门\":\"haimen|399\",\"东海\":\"donghai|2147\",\n" + + " \"扬中\":\"yangzhong|649\",\"兴化\":\"xinghuashi|699\",\"新沂\":\"xinyishi|478\",\"泰兴\":\"taixing|696\",\"如东\":\"rudong|402\",\n" + + " \"邳州\":\"pizhou|477\",\"沛县\":\"xzpeixian|11349\",\"靖江\":\"jingjiang|698\",\"建湖\":\"jianhu|618\",\"海安\":\"haian|401\",\n" + + " \"东台\":\"dongtai|615\",\"丹阳\":\"danyang|648\",\"宝应县\":\"baoyingx|14451\",\"灌南\":\"guannan|2150\",\"灌云\":\"guanyun|2148\",\n" + + " \"姜堰\":\"jiangyan|697\",\"金坛\":\"jintan|468\",\"昆山\":\"szkunshan|16\",\"泗洪\":\"sihong|5958\",\"泗阳\":\"siyang|5959\",\n" + + " \"句容\":\"jurong|650\",\"射阳\":\"sheyang|621\",\"阜宁\":\"funingxian|620\",\"响水\":\"xiangshui|619\",\"盱眙\":\"xuyi|976\",\n" + + " \"金湖\":\"jinhu|975\",\"江阴\":\"jiangyins|34984\"\n" + + " }\n" + + " ,\n" + + " \"江西\":{\n" + + " \"南昌\":\"nc|669\",\"赣州\":\"ganzhou|2363\",\"九江\":\"jj|2247\",\"宜春\":\"yichun|5709\",\"吉安\":\"ja|2364\",\n" + + " \"上饶\":\"sr|10120\",\"萍乡\":\"px|2248\",\"抚州\":\"fuzhou|10134\",\"景德镇\":\"jdz|2360\",\"新余\":\"xinyu|10115\",\n" + + " \"鹰潭\":\"yingtan|3209\",\"永新\":\"yxx|11077\",\"乐平\":\"lepingshi|9048\",\"进贤\":\"jinxian|677\",\"分宜\":\"fenyi|10118\",\n" + + " \"丰城\":\"fengchengshi|5711\",\"樟树\":\"zhangshu|5713\",\"高安\":\"gaoan|5712\",\"余江\":\"yujiang|3210\",\"南城\":\"nanchengx|10137\",\n" + + " \"浮梁\":\"fuliangxian|9071\"\n" + + " }\n" + + " ,\n" + + " \"吉林\":{\n" + + " \"长春\":\"cc|319\",\"吉林\":\"jl|700\",\"四平\":\"sp|10171\",\"延边\":\"yanbian|3184\",\"松原\":\"songyuan|2315\",\n" + + " \"白城\":\"bc|5918\",\"通化\":\"th|10159\",\"白山\":\"baishan|10179\",\"辽源\":\"liaoyuan|2501\",\"公主岭\":\"gongzhuling|10171\",\n" + + " \"梅河口\":\"meihekou|10162\",\"扶余\":\"fuyuxian|9085\",\"长岭\":\"changlingxian|9084\",\"桦甸\":\"huadian|706\",\"磐石\":\"panshi|708\",\n" + + " \"梨树县\":\"lishu|10176\"\n" + + " }\n" + + " ,\n" + + " \"辽宁\":{\n" + + " \"沈阳\":\"sy|188\",\"大连\":\"dl|147\",\"鞍山\":\"as|523\",\"锦州\":\"jinzhou|2354\",\"抚顺\":\"fushun|5722\",\n" + + " \"营口\":\"yk|5898\",\"盘锦\":\"pj|2041\",\"朝阳\":\"cy|10106\",\"丹东\":\"dandong|3445\",\"辽阳\":\"liaoyang|2038\",\n" + + " \"本溪\":\"benxi|5845\",\"葫芦岛\":\"hld|10088\",\"铁岭\":\"tl|6729\",\"阜新\":\"fx|10097\",\"庄河\":\"pld|3306\",\n" + + " \"瓦房店\":\"wfd|3279\",\"灯塔\":\"dengta|2071\",\"凤城\":\"fengcheng|3450\",\"北票\":\"beipiao|10109\",\"开原\":\"kaiyuan|6733\"\n" + + " }\n" + + " ,\n" + + " \"宁夏\":{\n" + + " \"银川\":\"yinchuan|2054\",\"吴忠\":\"wuzhong|9962\",\"石嘴山\":\"szs|9971\",\"中卫\":\"zw|9951\",\"固原\":\"guyuan|2421\"\n" + + " }\n" + + " ,\n" + + " \"内蒙古\":{\n" + + " \"呼和浩特\":\"hu|811\",\"包头\":\"bt|801\",\"赤峰\":\"chifeng|6700\",\"鄂尔多斯\":\"erds|2037\",\"通辽\":\"tongliao|10015\",\n" + + " \"呼伦贝尔\":\"hlbe|10039\",\"巴彦淖尔市\":\"bycem|10070\",\"乌兰察布\":\"wlcb|9993\",\"锡林郭勒\":\"xl|2408\",\"兴安盟\":\"xam|9976\",\n" + + " \"乌海\":\"wuhai|2404\",\"阿拉善盟\":\"alsm|10083\",\"海拉尔\":\"hlr|2043\"\n" + + " }\n" + + " ,\n" + + " \"青海\":{\n" + + " \"西宁\":\"xn|2052\",\"海西\":\"hx|9902\",\"海北\":\"haibei|9917\",\"果洛\":\"guoluo|9936\",\"海东\":\"haidong|9909\",\n" + + " \"黄南\":\"huangnan|9896\",\"玉树\":\"ys|9888\",\"海南\":\"hainan|10574\",\"格尔木\":\"geermushi|9904\"\n" + + " }\n" + + " ,\n" + + " \"山东\":{\n" + + " \"青岛\":\"qd|122\",\"济南\":\"jn|265\",\"烟台\":\"yt|228\",\"潍坊\":\"wf|362\",\"临沂\":\"linyi|505\",\n" + + " \"淄博\":\"zb|385\",\"济宁\":\"jining|450\",\"泰安\":\"ta|686\",\"聊城\":\"lc|882\",\"威海\":\"weihai|518\",\n" + + " \"枣庄\":\"zaozhuang|961\",\"德州\":\"dz|728\",\"日照\":\"rizhao|3177\",\"东营\":\"dy|623\",\"菏泽\":\"heze|5632\",\n" + + " \"滨州\":\"bz|944\",\"莱芜\":\"lw|2292\",\"章丘\":\"zhangqiu|8680\",\"垦利\":\"kl|11313\",\"诸城\":\"zc|9146\",\n" + + " \"寿光\":\"shouguang|369\",\"龙口\":\"longkou|233\",\"曹县\":\"caoxian|5638\",\"单县\":\"shanxian|5636\",\"肥城\":\"feicheng|690\",\n" + + " \"高密\":\"gaomi|371\",\"广饶\":\"guangrao|627\",\"桓台\":\"huantaixian|7335\",\"莒县\":\"juxian|3180\",\"莱州\":\"laizhou|235\",\n" + + " \"蓬莱\":\"penglai|237\",\"青州\":\"qingzhou|367\",\"荣成\":\"rongcheng|522\",\"乳山\":\"rushan|520\",\"滕州\":\"tengzhou|967\",\n" + + " \"新泰\":\"xintai|689\",\"招远\":\"zhaoyuan|3325\",\"邹城\":\"zoucheng|455\",\"邹平\":\"zouping|946\",\"临清\":\"linqing|884\",\n" + + " \"茌平\":\"chiping|887\",\"郓城\":\"hzyc|5637\",\"博兴\":\"boxing|949\",\"东明\":\"dongming|5641\",\"巨野\":\"juye|5640\",\n" + + " \"无棣\":\"wudi|951\",\"齐河\":\"qihe|734\",\"微山\":\"weishan|459\",\"禹城\":\"yuchengshi|731\",\"临邑\":\"linyixianq|739\",\n" + + " \"乐陵\":\"leling|730\",\"莱阳\":\"laiyang|234\",\"宁津\":\"ningjin|733\",\"高唐\":\"gaotang|885\",\"莘县\":\"shenxian|888\",\n" + + " \"阳谷\":\"yanggu|886\",\"冠县\":\"guanxian|890\",\"平邑\":\"pingyi|514\",\"郯城\":\"tancheng|510\",\"沂源\":\"yiyuanxian|7334\",\n" + + " \"汶上\":\"wenshang|460\",\"梁山\":\"liangshanx|462\",\"利津\":\"lijin|628\",\"沂南\":\"yinanxian|7301\",\"栖霞\":\"qixia|238\",\n" + + " \"宁阳\":\"ningyang|691\",\"东平\":\"dongping|692\",\"昌邑\":\"changyishi|372\",\"安丘\":\"anqiu|370\",\"昌乐\":\"changle|373\",\n" + + " \"临朐\":\"linqu|374\",\"鄄城\":\"juancheng|5635\"\n" + + " }\n" + + " ,\n" + + " \"山西\":{\n" + + " \"太原\":\"ty|740\",\"临汾\":\"linfen|5669\",\"大同\":\"dt|6964\",\"运城\":\"yuncheng|5653\",\"晋中\":\"jz|8854\",\n" + + " \"长治\":\"changzhi|6921\",\"晋城\":\"jincheng|3350\",\"阳泉\":\"yq|8760\",\"吕梁\":\"lvliang|3222\",\"忻州\":\"xinzhou|3453\",\n" + + " \"朔州\":\"shuozhou|9871\",\"临猗\":\"linyixian|9193\",\"清徐\":\"qingxu|10908\",\"柳林\":\"liulin|3225\",\"高平\":\"gaoping|3354\",\n" + + " \"泽州\":\"zezhou|3353\",\"襄垣\":\"xiangyuanxian|6928\",\"孝义\":\"xiaoyi|3227\"\n" + + " }\n" + + " ,\n" + + " \"陕西\":{\n" + + " \"西安\":\"xa|483\",\"咸阳\":\"xianyang|7453\",\"宝鸡\":\"baoji|2044\",\"渭南\":\"wn|5733\",\"汉中\":\"hanzhong|3163\",\n" + + " \"榆林\":\"yl|5942\",\"延安\":\"yanan|8973\",\"安康\":\"ankang|3157\",\"商洛\":\"sl|9854\",\"铜川\":\"tc|9832\",\n" + + " \"神木\":\"shenmu|5944\",\"韩城\":\"hancheng|5735\",\"府谷\":\"fugu|5945\",\"靖边\":\"jingbian|5947\",\"定边\":\"dingbian|5948\"\n" + + " }\n" + + " ,\n" + + " \"四川\":{\n" + + " \"成都\":\"cd|102\",\"绵阳\":\"mianyang|1057\",\"德阳\":\"deyang|2373\",\"南充\":\"nanchong|2378\",\"宜宾\":\"yb|2380\",\n" + + " \"自贡\":\"zg|6745\",\"乐山\":\"ls|3237\",\"泸州\":\"luzhou|2372\",\"达州\":\"dazhou|9799\",\"内江\":\"scnj|5928\",\n" + + " \"遂宁\":\"suining|9688\",\"攀枝花\":\"panzhihua|2371\",\"眉山\":\"ms|9704\",\"广安\":\"ga|2381\",\"资阳\":\"zy|6803\",\n" + + " \"凉山\":\"liangshan|9717\",\"广元\":\"guangyuan|9749\",\"雅安\":\"ya|9687\",\"巴中\":\"bazhong|9811\",\"阿坝\":\"ab|9817\",\n" + + " \"甘孜\":\"ganzi|9764\",\"安岳\":\"anyuexian|6806\",\"广汉\":\"guanghanshi|8719\",\"简阳\":\"jianyangshi|6805\",\"仁寿\":\"renshouxian|9706\",\n" + + " \"射洪\":\"shehongxian|9694\",\"大竹\":\"dazu|9806\",\"宣汉\":\"xuanhan|9804\",\"渠县\":\"qux|9807\",\"长宁\":\"changningx|7148\"\n" + + " }\n" + + " ,\n" + + " \"新疆\":{\n" + + " \"乌鲁木齐\":\"xj|984\",\"昌吉\":\"changji|8582\",\"巴音郭楞\":\"bygl|9530\",\"伊犁\":\"yili|9472\",\"阿克苏\":\"aks|9499\",\n" + + " \"喀什\":\"ks|9326\",\"哈密\":\"hami|7452\",\"克拉玛依\":\"klmy|2042\",\"博尔塔拉\":\"betl|9529\",\"吐鲁番\":\"tlf|9475\",\n" + + " \"和田\":\"ht|9489\",\"石河子\":\"shz|9551\",\"克孜勒苏\":\"kzls|9519\",\"阿拉尔\":\"ale|9539\",\"五家渠\":\"wjq|9562\",\n" + + " \"图木舒克\":\"tmsk|9559\",\"库尔勒\":\"kel|7168\",\"阿勒泰\":\"alt|18837\",\"塔城\":\"tac|18845\"\n" + + " }\n" + + " ,\n" + + " \"西藏\":{\n" + + " \"拉萨\":\"lasa|2055\",\"日喀则\":\"rkz|9615\",\"山南\":\"sn|9576\",\"林芝\":\"linzhi|9646\",\"昌都\":\"changdu|9648\",\n" + + " \"那曲\":\"nq|9618\",\"阿里\":\"al|9678\",\"日土\":\"rituxian|9682\",\"改则\":\"gaizexian|9684\"\n" + + " }\n" + + " ,\n" + + " \"云南\":{\n" + + " \"昆明\":\"km|541\",\"曲靖\":\"qj|2389\",\"大理\":\"dali|2398\",\"红河\":\"honghe|2394\",\"玉溪\":\"yx|2040\",\n" + + " \"丽江\":\"lj|2392\",\"文山\":\"ws|2395\",\"楚雄\":\"cx|2393\",\"西双版纳\":\"bn|2397\",\"昭通\":\"zt|9409\",\n" + + " \"德宏\":\"dh|9437\",\"普洱\":\"pe|9444\",\"保山\":\"bs|2390\",\"临沧\":\"lincang|9422\",\"迪庆\":\"diqing|9432\",\n" + + " \"怒江\":\"nujiang|9462\",\"弥勒\":\"milexian|8892\",\"安宁\":\"anningshi|547\",\"宣威\":\"xuanwushi|7533\"\n" + + " }\n" + + " ,\n" + + " \"浙江\":{\n" + + " \"杭州\":\"hz|79\",\"宁波\":\"nb|135\",\"温州\":\"wz|330\",\"金华\":\"jh|531\",\"嘉兴\":\"jx|497\",\n" + + " \"台州\":\"tz|403\",\"绍兴\":\"sx|355\",\"湖州\":\"huzhou|831\",\"丽水\":\"lishui|7921\",\"衢州\":\"quzhou|6793\",\n" + + " \"舟山\":\"zhoushan|8481\",\"乐清\":\"yueqingcity|13950\",\"瑞安\":\"ruiancity|13951\",\"义乌\":\"yiwu|12291\",\"余姚\":\"yuyao|5333\",\n" + + " \"诸暨\":\"zhuji|357\",\"象山\":\"xiangshanxian|6738\",\"温岭\":\"wenling|408\",\"桐乡\":\"tongxiang|502\",\"慈溪\":\"cixi|5334\",\n" + + " \"长兴\":\"changxing|834\",\"嘉善\":\"jiashanx|14357\",\"海宁\":\"haining|500\",\"德清\":\"deqing|835\",\"东阳\":\"dongyang|536\",\n" + + " \"安吉\":\"anji|836\",\"苍南\":\"cangnanxian|7579\",\"临海\":\"linhai|407\",\"永康\":\"yongkang|537\",\"玉环\":\"yuhuan|409\",\n" + + " \"平湖\":\"pinghushi|501\",\"海盐\":\"haiyan|504\",\"武义县\":\"wuyix|14528\",\"嵊州\":\"shengzhou|359\",\"新昌\":\"xinchang|361\",\n" + + " \"江山\":\"jiangshanshi|6796\",\"平阳\":\"pingyangxian|7575\"\n" + + " }\n" + + " ,\n" + + " \"其他\":{\n" + + " \"中国香港\":\"hk|2050\",\"中国澳门\":\"am|9399\",\"中国台湾\":\"tw|2051\"\n" + + " }\n" + + " ,\n" + + " \"海外\":{\n" + + " \"洛杉矶\":\"gllosangeles\",\"旧金山\":\"glsanfrancisco\",\"纽约\":\"glnewyork\",\"多伦多\":\"gltoronto\",\"温哥华\":\"glvancouver\",\n" + + " \"伦敦\":\"glgreaterlondon\",\"莫斯科\":\"glmoscow\",\"首尔\":\"glseoul\",\"东京\":\"gltokyo\",\"新加坡\":\"glsingapore\",\n" + + " \"曼谷\":\"glbangkok\",\"清迈\":\"glchiangmai\",\"迪拜\":\"gldubai\",\"奥克兰\":\"glauckland\",\"悉尼\":\"glsydney\",\n" + + " \"墨尔本\":\"glmelbourne\",\"其他海外城市\":\"city\"\n" + + " }\n" + + " }"; + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + JSONObject jsonObject = JSONObject.parseObject(cityList) + Set provinceSet = jsonObject.keySet() + List ret = new ArrayList<>() + for (String province: provinceSet) { + JSONObject provinceInfo = jsonObject[province] + for (String city : provinceInfo.keySet()) { + def city_short = provinceInfo[city].split("\\|")[0] + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", city_short) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "58.com") + ret.add(dbObject) + } + } + cityConn.insert(ret) + print("58城市数据写入成功") + } +} From 66058f0a7152bc5c70f5d4348486c827e46977f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 10 May 2021 18:46:34 +0800 Subject: [PATCH 020/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/TaskExport.groovy | 104 +++++------------- .../fangjia/script/statistics/ApiUtil.java | 4 +- .../script/statistics/TaskPersonDTO.java | 31 ++++++ 3 files changed, 61 insertions(+), 78 deletions(-) create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index f2e64dd9e..196e9950b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -3,6 +3,7 @@ import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel import com.fangjia.script.statistics.Task import com.fangjia.script.statistics.TaskDetail +import com.fangjia.script.statistics.TaskPersonDTO import org.apache.commons.collections.CollectionUtils /** @@ -14,89 +15,40 @@ import org.apache.commons.collections.CollectionUtils class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01" - static final String endTime = "2021-04-30" + static final String startTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { - def sortedResult = [] - sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + List list = new ArrayList<>() def result = [] - List projects = ApiUtil.listProject(); - for (Project project : projects) { + result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] + List tasks = ApiUtil.listTaskByPerson(startTime, endTime) + for (Task task: tasks) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + + TaskPersonDTO dto = new TaskPersonDTO() + dto.name = task.response.name + dto.project = "" + dto.taskName = task.name + dto.score = Double.valueOf(taskDetail.flowConten) + dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) + + list.add(dto) + } + + Collections.sort(list) + + for (TaskPersonDTO dto : list) { def ret = [] - List labels = ApiUtil.listProjectLabel(project.getId()) - ret[2] = project.getName() - for (ProjectLabel label : labels) { - if ("商务负责人" == label.getGroups().getName()) { - if (ret[3] == null) { - ret[3] = label.getName() - } else { - ret[3] = ret[3] + "、" + label.getName() - } - } else if ("项目类别" == label.getGroups().getName()) { - ret[0] = label.getName() - } - } - // 获取项目下任务 - List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) - Double totalTime = 0D; - if (CollectionUtils.isNotEmpty(tasks)) { - for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - boolean find = false - for (int i = 4; i < sortedResult[0].size(); i++) { - // 找到该项目下该负责人,则加工时 - if (sortedResult[0][i] == taskDetail.getResponse().getName()) { - if (ret[i] == null) { - ret[i] = Double.valueOf(taskDetail.getFlowConten()) - } else { - ret[i] += Double.valueOf(taskDetail.getFlowConten()) - } - find = true - } - } - // 未找到,新增姓名 - if (!find) { - sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() - ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) - } - } - } + ret[0] = dto.name; + ret[1] = dto.project; + ret[2] = dto.taskName; + ret[3] = dto.score; + ret[4] = dto.workTime; result << ret } - def allTotal = [] - for (def label : sort) { - def total = [] - for (def ret : result) { - if (ret[0] == label) { - sortedResult << ret - for (int i = 4; i < ret.size(); i++) { - if (total[i] == null) { - total[i] = ret[i] == null ? 0 : Double.valueOf(ret[i]) - } else { - total[i] += (ret[i] == null ? 0 : Double.valueOf(ret[i])) - } - } - } - } - total[1] = 0 - total[2] = "小计" - sortedResult << total - for (int i = 4; i < total.size(); i++) { - if (allTotal[i] == null) { - allTotal[i] = total[i] == null ? 0 : Double.valueOf(total[i]) - } else { - allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) - } - total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) - } - } - allTotal[2] = "人员绩效总计" - sortedResult << allTotal - - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计"); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 33c96ae42..10a2f60f4 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -335,8 +335,8 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - //System.out.println(ApiUtil.listTaskByProject("ecd77309ab1c4b83b54ee87b8ddd4f98", "2021-04-01", "2021-04-30")); + System.out.println(ApiUtil.listTaskByProject("d00fcb9c5307499eab8f9798a23f6360", "2021-04-01", "2021-04-30")); //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); - System.out.println(ApiUtil.listTaskByPerson("2021-04-01", "2021-04-30")); + //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java new file mode 100644 index 000000000..d22c4a364 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/TaskPersonDTO.java @@ -0,0 +1,31 @@ +package com.fangjia.script.statistics; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @author shenjian + * @since 2021/5/10 + */ +@NoArgsConstructor +@Data +public class TaskPersonDTO implements Comparable { + + private String name; + private String project; + private String taskName; + private Double score; + private Double workTime; + + @Override + public int compareTo(TaskPersonDTO o2) { + if (this.getName().equals(o2.getName())) { + if (this.getProject() != null) { + return this.getProject().compareTo(o2.getProject()); + } + } else { + return this.getName().compareTo(o2.getName()); + } + return 0; + } +} From 44351a80214e1c37810722f1cfaa48cb4eda22ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 10:22:53 +0800 Subject: [PATCH 021/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B9=E7=9B=AE?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=8D=95=E5=85=83=E6=A0=BC=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 10 ++++- .../com/fangjia/script/util/ExcelUtils.groovy | 37 ++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 76ea84abe..d08846701 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -18,6 +18,7 @@ class ProjectExport { static final String endTime = "2021-04-30" static void main(String[] args) { + List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] @@ -57,14 +58,21 @@ class ProjectExport { def sortedResult = [] sortedResult << ['项目类别', '项目名称', '项目负责人', '总工时(天)'] for (def label : sort) { + List mergeRowCel = new ArrayList<>() + mergeRowCel.add(sortedResult.size()) for (def ret : result) { if (ret[0] == label) { sortedResult << ret } } + mergeRowCel.add(sortedResult.size() - 1) + mergeRowCel.add(0) + mergeRowCel.add(0) + + mergeData.add(mergeRowCel) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 4cad85c53..203508a51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -1,11 +1,14 @@ package com.fangjia.script.util +import org.apache.commons.collections.CollectionUtils import org.apache.commons.lang.StringUtils import org.apache.poi.hssf.record.BoolErrRecord - +import org.apache.poi.hssf.usermodel.HSSFBorderFormatting import org.apache.poi.hssf.usermodel.HSSFWorkbook import org.apache.poi.ss.formula.eval.ErrorEval import org.apache.poi.ss.usermodel.* +import org.apache.poi.ss.util.CellRangeAddress +import org.apache.poi.ss.util.RegionUtil import org.apache.poi.xssf.streaming.SXSSFWorkbook import java.util.concurrent.ExecutorService @@ -235,7 +238,7 @@ public class ExcelUtils{ return super.getProperty(propertyName) } - public static void write(String fileName, List> data, String sheetName){ + public static void write(String fileName, List> data, String sheetName, List> mergeData){ File file = new File(fileName) File dir = file.getParentFile(); @@ -304,6 +307,8 @@ public class ExcelUtils{ throw e } } + mergeCell(workBook, mainSheet, mergeData) + for(int i = 0; i < maxColumnNum; i++ ){ mainSheet.autoSizeColumn(i) } @@ -321,6 +326,34 @@ public class ExcelUtils{ fos.close(); } + static void write(String fileName, List> data, String sheetName){ + write(fileName, data, sheetName, null) + } + + /** + * 合并单元格 + * @param workBook + * @param sheet + * @param data int firstRow, int lastRow, int firstCol, int lastCol + */ + static mergeCell(Workbook workBook, Sheet sheet, List> data) { + if (CollectionUtils.isEmpty(data)) { + return + } + //设置style + CellStyle mergedCellStyle = workBook.createCellStyle() + mergedCellStyle.verticalAlignment = VerticalAlignment.CENTER; + mergedCellStyle.alignment = HorizontalAlignment.CENTER; + + for (List rowCell : data) { + CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3)) + sheet.addMergedRegion(cra); + //设置合并后style + def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2)); + cell.setCellStyle(mergedCellStyle) + } + } + public static newWorkBook(String fileName){ Workbook wb; if(fileName.endsWith(".xlsx")){ From 9c8d0e33a77b69c3953b32260038312ec055765f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 10:42:45 +0800 Subject: [PATCH 022/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=A9=E6=95=88?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=8D=95=E5=85=83=E6=A0=BC=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ScoreExport.groovy | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index baead79bc..710ebd395 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -14,10 +14,11 @@ import org.apache.commons.collections.CollectionUtils class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01" - static final String endTime = "2021-04-30" + static final String startTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { + List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; @@ -40,7 +41,6 @@ class ScoreExport { } // 获取项目下任务 List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) - Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) @@ -68,6 +68,7 @@ class ScoreExport { def allTotal = [] for (def label : sort) { def total = [] + int startRow = sortedResult.size() for (def ret : result) { if (ret[0] == label) { sortedResult << ret @@ -80,8 +81,22 @@ class ScoreExport { } } } + int endRow = sortedResult.size() - 1 + List mergeRowCel = new ArrayList<>() + mergeRowCel.add(startRow) + mergeRowCel.add(endRow) + mergeRowCel.add(0) + mergeRowCel.add(0) + mergeData.add(mergeRowCel) - total[1] = 0 + mergeRowCel = new ArrayList<>() + mergeRowCel.add(startRow) + mergeRowCel.add(endRow) + mergeRowCel.add(1) + mergeRowCel.add(1) + mergeData.add(mergeRowCel) + + double totalNow = 0 total[2] = "小计" sortedResult << total for (int i = 4; i < total.size(); i++) { @@ -90,13 +105,14 @@ class ScoreExport { } else { allTotal[i] += total[i] == null ? 0 : Double.valueOf(total[i]) } - total[1] += total[i] == null ? 0 : Double.valueOf(total[i]) + totalNow += total[i] == null ? 0 : Double.valueOf(total[i]) } + sortedResult[startRow][1] = totalNow } allTotal[2] = "人员绩效总计" sortedResult << allTotal - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计"); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计", mergeData); } } From dbcf202d226d754e45801d4306173cfdc2a34a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 11:28:17 +0800 Subject: [PATCH 023/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E7=BB=9F=E8=AE=A1=E5=B0=8F=E8=AE=A1=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 4 +- .../shenjian/statistics/TaskExport.groovy | 41 ++++++++++++++++++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index d08846701..317fd8fdb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -14,8 +14,8 @@ import org.apache.commons.collections.CollectionUtils class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01" - static final String endTime = "2021-04-30" + static final String startTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 196e9950b..7d05209bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -19,6 +19,7 @@ class TaskExport { static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { + List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] @@ -38,6 +39,7 @@ class TaskExport { Collections.sort(list) + Map> map = new HashMap<>() for (TaskPersonDTO dto : list) { def ret = [] ret[0] = dto.name; @@ -45,10 +47,47 @@ class TaskExport { ret[2] = dto.taskName; ret[3] = dto.score; ret[4] = dto.workTime; + if (map.containsKey(dto.name)) { + Map info = map.get(dto.name) + info.put("endRow", Double.valueOf(info.get("endRow") + 1)) + info.put("totalScore", Double.valueOf(info.get("totalScore") + dto.score)) + info.put("workTime", Double.valueOf(info.get("workTime") + dto.workTime)) + map.put(dto.name, info); + } else { + if (result.size() > 1) { + def total = [] + total[2] = "小计" + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + result << total + } + Map info = new HashMap<>() + info.put("startRow", Double.valueOf(result.size())) + info.put("endRow", Double.valueOf(result.size())) + info.put("startCell", 0D) + info.put("endCell", 0D) + info.put("totalScore", Double.valueOf(dto.score)) + info.put("workTime", Double.valueOf(dto.workTime)) + map.put(dto.name, info) + } result << ret } + def total = [] + total[2] = "小计" + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + result << total - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计"); + for (Map info : map.values()) { + List data = new ArrayList<>() + data.add(info.get("startRow").toInteger()) + data.add(info.get("endRow").toInteger()) + data.add(info.get("startCell").toInteger()) + data.add(info.get("endCell").toInteger()) + mergeData.add(data) + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计", mergeData); } } From 87c938d283fa613ae79500a9a6312ebfc4662299 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 13:39:54 +0800 Subject: [PATCH 024/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 20 ++++++---- .../shenjian/statistics/ScoreExport.groovy | 38 +++++++++++-------- .../shenjian/statistics/TaskExport.groovy | 38 ++++++++++++++----- .../fangjia/script/statistics/ApiUtil.java | 6 +-- 4 files changed, 66 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 317fd8fdb..abff40879 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -15,7 +15,8 @@ class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -43,12 +44,17 @@ class ProjectExport { Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - // 先取绩效,不存在则取工时 - if (taskDetail.getFlowConten() != null) { - totalTime += Double.valueOf(taskDetail.getFlowConten()) - } else { - totalTime += Double.valueOf(taskDetail.getWorkTime()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 先取绩效,不存在则取工时 + if (taskDetail.getFlowConten() != null) { + totalTime += Double.valueOf(taskDetail.getFlowConten()) + } else { + totalTime += Double.valueOf(taskDetail.getWorkTime()) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index 710ebd395..ef71d71de 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -15,7 +15,8 @@ class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -43,23 +44,28 @@ class ScoreExport { List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - boolean find = false - for (int i = 4; i < sortedResult[0].size(); i++) { - // 找到该项目下该负责人,则加工时 - if (sortedResult[0][i] == taskDetail.getResponse().getName()) { - if (ret[i] == null) { - ret[i] = Double.valueOf(taskDetail.getFlowConten()) - } else { - ret[i] += Double.valueOf(taskDetail.getFlowConten()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true } - find = true } - } - // 未找到,新增姓名 - if (!find) { - sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() - ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 7d05209bc..dcc5fcccd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -16,25 +16,43 @@ class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] - List tasks = ApiUtil.listTaskByPerson(startTime, endTime) + int limitStart = 0 + List tasks = new ArrayList<>() + List tempTasks = tasks; + + while (tempTasks.isEmpty() || tempTasks.size() == 50) { + tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + limitStart += 50 + tasks.addAll(tempTasks) + } for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } - TaskPersonDTO dto = new TaskPersonDTO() - dto.name = task.response.name - dto.project = "" - dto.taskName = task.name - dto.score = Double.valueOf(taskDetail.flowConten) - dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) + TaskPersonDTO dto = new TaskPersonDTO() + dto.name = task.response.name + dto.project = "" + dto.taskName = task.name + dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten) + dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) - list.add(dto) + list.add(dto) + } } Collections.sort(list) diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 10a2f60f4..a21653dce 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -270,12 +270,12 @@ public class ApiUtil { /** * 鑾峰彇浜哄憳浠诲姟鍒楄〃 */ - public static List listTaskByPerson(String startTime, String endTime) { + public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); - body.put("limitStart", "0"); - body.put("limitNum", "50"); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); //body.put("userid", ""); body.put("type", ""); body.put("timezone", ""); From a8c32d7276c67703b3e29693ca028c9d849b2c1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 13:39:54 +0800 Subject: [PATCH 025/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 22 +++++---- .../shenjian/statistics/ScoreExport.groovy | 48 ++++++++++++------- .../shenjian/statistics/TaskExport.groovy | 40 +++++++++++----- .../fangjia/script/statistics/ApiUtil.java | 9 ++-- 4 files changed, 78 insertions(+), 41 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 317fd8fdb..f1914cadd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -15,7 +15,8 @@ class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -43,12 +44,17 @@ class ProjectExport { Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - // 先取绩效,不存在则取工时 - if (taskDetail.getFlowConten() != null) { - totalTime += Double.valueOf(taskDetail.getFlowConten()) - } else { - totalTime += Double.valueOf(taskDetail.getWorkTime()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 先取绩效,不存在则取工时 + if (taskDetail.getFlowConten() != null) { + totalTime += Double.valueOf(taskDetail.getFlowConten()) + } else { + totalTime += Double.valueOf(taskDetail.getWorkTime()) + } } } } @@ -72,7 +78,7 @@ class ProjectExport { mergeData.add(mergeRowCel) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\项目统计.xlsx", sortedResult, "项目统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index 710ebd395..f5a7287a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -15,7 +15,8 @@ class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() @@ -39,27 +40,38 @@ class ScoreExport { ret[0] = label.getName() } } + if (project.getName() == "建行楼盘数据治理项目") { + println('建行楼盘数据治理项目') + } // 获取项目下任务 List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - boolean find = false - for (int i = 4; i < sortedResult[0].size(); i++) { - // 找到该项目下该负责人,则加工时 - if (sortedResult[0][i] == taskDetail.getResponse().getName()) { - if (ret[i] == null) { - ret[i] = Double.valueOf(taskDetail.getFlowConten()) - } else { - ret[i] += Double.valueOf(taskDetail.getFlowConten()) - } - find = true + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + boolean find = false + if (taskDetail.getResponse().getName() == '裴银银') { + println('裴银银') + } + for (int i = 4; i < sortedResult[0].size(); i++) { + // 找到该项目下该负责人,则加工时 + if (sortedResult[0][i] == taskDetail.getResponse().getName()) { + if (ret[i] == null) { + ret[i] = Double.valueOf(taskDetail.getFlowConten()) + } else { + ret[i] += Double.valueOf(taskDetail.getFlowConten()) + } + find = true + } + } + // 未找到,新增姓名 + if (!find) { + sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() + ret[sortedResult[0].size() - 1] = Double.valueOf(taskDetail.getFlowConten()) } - } - // 未找到,新增姓名 - if (!find) { - sortedResult[0][sortedResult[0].size()] = taskDetail.getResponse().getName() - ret[ret.size()] = Double.valueOf(taskDetail.getFlowConten()) } } } @@ -112,7 +124,7 @@ class ScoreExport { allTotal[2] = "人员绩效总计" sortedResult << allTotal - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\绩效统计.xlsx", sortedResult, "绩效统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 7d05209bc..182b34019 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -16,25 +16,43 @@ class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" - static final String endTime = "2021-04-30 23:59:59" + static final String monthEndTime = "2021-04-30 23:59:59" + static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] - List tasks = ApiUtil.listTaskByPerson(startTime, endTime) + int limitStart = 0 + List tasks = new ArrayList<>() + List tempTasks = tasks; + + while (tempTasks.isEmpty() || tempTasks.size() == 50) { + tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + limitStart += 50 + tasks.addAll(tempTasks) + } for (Task task: tasks) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + // 获取计划时间在当月的任务 + if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) + || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) + ) { + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } - TaskPersonDTO dto = new TaskPersonDTO() - dto.name = task.response.name - dto.project = "" - dto.taskName = task.name - dto.score = Double.valueOf(taskDetail.flowConten) - dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) + TaskPersonDTO dto = new TaskPersonDTO() + dto.name = task.response.name + dto.project = "" + dto.taskName = task.name + dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten) + dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) - list.add(dto) + list.add(dto) + } } Collections.sort(list) @@ -87,7 +105,7 @@ class TaskExport { mergeData.add(data) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\任务统计.xlsx", result, "任务统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 10a2f60f4..1de09a0c9 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -270,12 +270,12 @@ public class ApiUtil { /** * 鑾峰彇浜哄憳浠诲姟鍒楄〃 */ - public static List listTaskByPerson(String startTime, String endTime) { + public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); - body.put("limitStart", "0"); - body.put("limitNum", "50"); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); //body.put("userid", ""); body.put("type", ""); body.put("timezone", ""); @@ -335,7 +335,8 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - System.out.println(ApiUtil.listTaskByProject("d00fcb9c5307499eab8f9798a23f6360", "2021-04-01", "2021-04-30")); + System.out.println(ApiUtil.listTaskByProject("457f11e89aad42b6a7bb289e8341ac04", "2021-04-01", "2021-05-15")); + System.out.println(ApiUtil.listTaskByProject("41bdf23cbf634035b694f92ba72b067f", "2021-04-01", "2021-05-15")); //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); } From 1e98b447d4daf4f932b5992887272d296a23fe13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 15:48:15 +0800 Subject: [PATCH 026/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A0=E6=A0=87?= =?UTF-8?q?=E7=AD=BE=E9=A1=B9=E7=9B=AE=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 14 +++++++++++-- .../shenjian/statistics/ScoreExport.groovy | 21 ++++++++++++------- .../fangjia/script/statistics/ApiUtil.java | 13 ++++++------ 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index f1914cadd..33ffdfee3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -20,7 +20,7 @@ class ProjectExport { static void main(String[] args) { List> mergeData = new ArrayList<>() - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; def result = [] List projects = ApiUtil.listProject(); @@ -40,7 +40,17 @@ class ProjectExport { } } // 获取项目总工时 - List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + int limitStart = 0 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + List tempTasks = tasks; + + while (tasks != null && tempTasks != null && tasks.size() == 50) { + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + limitStart += 50 + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } + } Double totalTime = 0D; if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index f5a7287a6..ce333aaa7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -22,7 +22,7 @@ class ScoreExport { List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; def result = [] List projects = ApiUtil.listProject(); for (Project project : projects) { @@ -40,11 +40,18 @@ class ScoreExport { ret[0] = label.getName() } } - if (project.getName() == "建行楼盘数据治理项目") { - println('建行楼盘数据治理项目') - } // 获取项目下任务 - List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime) + int limitStart = 0 + List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + List tempTasks = tasks; + + while (tasks != null && tempTasks != null && tasks.size() == 50) { + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + limitStart += 50 + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } + } if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { // 获取计划时间在当月的任务 @@ -53,8 +60,8 @@ class ScoreExport { ) { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) boolean find = false - if (taskDetail.getResponse().getName() == '裴银银') { - println('裴银银') + if (taskDetail.getResponse().getName() == '穆良书' && taskDetail.getName() == "模型级联算法方案设计") { + println('穆良书') } for (int i = 4; i < sortedResult[0].size(); i++) { // 找到该项目下该负责人,则加工时 diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 1de09a0c9..40cdb1790 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -236,12 +236,12 @@ public class ApiUtil { * * @return */ - public static List listTaskByProject(String projectId, String startTime, String endTime) { + public static List listTaskByProject(String projectId, String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); - body.put("limitStart", "0"); - body.put("limitNum", "50"); + body.put("limitStart", limitStart); + body.put("limitNum", limitNum); body.put("projectId", projectId); body.put("createtimeStart", startTime); body.put("createtimeEnd", endTime); @@ -335,9 +335,10 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - System.out.println(ApiUtil.listTaskByProject("457f11e89aad42b6a7bb289e8341ac04", "2021-04-01", "2021-05-15")); - System.out.println(ApiUtil.listTaskByProject("41bdf23cbf634035b694f92ba72b067f", "2021-04-01", "2021-05-15")); - //System.out.println(ApiUtil.getTaskDetail("3bbb043ad0b44ea2a1213e725274e3ad")); + System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); + ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); + //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); + //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); } } From d1b5c9361e57318fa56fbb236fb2cc9e2111d029 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 16:03:45 +0800 Subject: [PATCH 027/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=A0=E6=A0=87?= =?UTF-8?q?=E7=AD=BE=E9=A1=B9=E7=9B=AE=E6=98=BE=E7=A4=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/statistics/DingTalkUtils.groovy | 3 +-- .../temp/shenjian/statistics/ProjectExport.groovy | 10 +++++++++- .../script/temp/shenjian/statistics/ScoreExport.groovy | 8 ++++++-- .../script/temp/shenjian/statistics/TaskExport.groovy | 4 ++++ 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy index f49025b41..19efb9171 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy @@ -1,3 +1,4 @@ + /** * 钉钉接口调用工具类 * @@ -6,8 +7,6 @@ */ class DingTalkUtils { - final String key = "ding0243a5c51f1f4aa8"; - static void main(String[] args) { } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 33ffdfee3..8c8dc6f64 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -19,8 +19,12 @@ class ProjectExport { static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { + export() + } + + public static void export() { List> mergeData = new ArrayList<>() - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] List projects = ApiUtil.listProject(); @@ -59,6 +63,10 @@ class ProjectExport { || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) ) { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } // 先取绩效,不存在则取工时 if (taskDetail.getFlowConten() != null) { totalTime += Double.valueOf(taskDetail.getFlowConten()) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index ce333aaa7..dcef469bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -11,7 +11,7 @@ import org.apache.commons.collections.CollectionUtils * @author shenjian * @since 2021/5/10 */ -class ScoreExport { +public class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-04-01 00:00:00" @@ -19,10 +19,14 @@ class ScoreExport { static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { + export() + } + + static void export() { List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] - def sort = ["建行项目", "基础建设", "非农行项目", "农行项目", null]; + def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] List projects = ApiUtil.listProject(); for (Project project : projects) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 182b34019..fc5bf1190 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -20,6 +20,10 @@ class TaskExport { static final String endTime = "2021-05-15 23:59:59" static void main(String[] args) { + export() + } + + static void export() { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] From 74f56daa84e580fc0e10db4ff471f8deabbf9728 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 17:15:08 +0800 Subject: [PATCH 028/212] =?UTF-8?q?=E6=8C=89=E5=AE=9E=E9=99=85=E5=AE=8C?= =?UTF-8?q?=E6=88=90=E6=97=B6=E9=97=B4=E7=BB=9F=E8=AE=A1=E7=BB=A9=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 30 +++++++---------- .../shenjian/statistics/ScoreExport.groovy | 19 ++++++----- .../shenjian/statistics/TaskExport.groovy | 23 +++++++------ .../fangjia/script/statistics/ApiUtil.java | 33 ++++++++++++++++--- 4 files changed, 61 insertions(+), 44 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 8c8dc6f64..2bd88e38b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -14,15 +14,15 @@ import org.apache.commons.collections.CollectionUtils class ProjectExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01 00:00:00" - static final String monthEndTime = "2021-04-30 23:59:59" - static final String endTime = "2021-05-15 23:59:59" + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { export() } - public static void export() { + static void export() { List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] @@ -48,9 +48,9 @@ class ProjectExport { List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) List tempTasks = tasks; - while (tasks != null && tempTasks != null && tasks.size() == 50) { - tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { limitStart += 50 + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) if (CollectionUtils.isNotEmpty(tempTasks)) { tasks.addAll(tempTasks) } @@ -59,22 +59,16 @@ class ProjectExport { if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { // 获取计划时间在当月的任务 - if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) - || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) - ) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - if (taskDetail == null) { - print("当前任务taskDetail为空:" + task) - continue; - } - // 先取绩效,不存在则取工时 + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + print("当前任务taskDetail为空:" + task) + continue; + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) if (taskDetail.getFlowConten() != null) { totalTime += Double.valueOf(taskDetail.getFlowConten()) - } else { - totalTime += Double.valueOf(taskDetail.getWorkTime()) } } - } } ret[3] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, totalTime) result << ret diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index dcef469bd..81b26e748 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -14,9 +14,9 @@ import org.apache.commons.collections.CollectionUtils public class ScoreExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01 00:00:00" - static final String monthEndTime = "2021-04-30 23:59:59" - static final String endTime = "2021-05-15 23:59:59" + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { export() @@ -49,9 +49,9 @@ public class ScoreExport { List tasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) List tempTasks = tasks; - while (tasks != null && tempTasks != null && tasks.size() == 50) { - tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { limitStart += 50 + tempTasks = ApiUtil.listTaskByProject(project.getId(), startTime, endTime, limitStart, 50) if (CollectionUtils.isNotEmpty(tempTasks)) { tasks.addAll(tempTasks) } @@ -59,10 +59,11 @@ public class ScoreExport { if (CollectionUtils.isNotEmpty(tasks)) { for (Task task: tasks) { // 获取计划时间在当月的任务 - if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) - || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) - ) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + continue; + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { boolean find = false if (taskDetail.getResponse().getName() == '穆良书' && taskDetail.getName() == "模型级联算法方案设计") { println('穆良书') diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index fc5bf1190..57e76d265 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -15,9 +15,9 @@ import org.apache.commons.collections.CollectionUtils class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; - static final String startTime = "2021-04-01 00:00:00" - static final String monthEndTime = "2021-04-30 23:59:59" - static final String endTime = "2021-05-15 23:59:59" + static final String startTime = "2021-03-01 00:00:00" + static final String monthStartTime = "2021-04-01 00:00:00" + static final String endTime = "2021-04-30 23:59:59" static void main(String[] args) { export() @@ -39,15 +39,14 @@ class TaskExport { } for (Task task: tasks) { // 获取计划时间在当月的任务 - if ((org.apache.commons.lang3.StringUtils.isNotBlank(task.getPlanBeginTime()) && monthEndTime > task.getPlanBeginTime()) - || (org.apache.commons.lang3.StringUtils.isBlank(task.getPlanBeginTime()) && task.getCreateDate() < monthEndTime) - ) { - TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) - if (taskDetail == null) { - print("当前任务taskDetail为空:" + task) - continue; - } - + TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) + if (taskDetail == null) { + continue; + } + if ("穆良书" == task.getResponse().getName() && task.getId() == "e3f81b0381b44696b06fb6242a44902f") { + print("穆良书") + } + if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { TaskPersonDTO dto = new TaskPersonDTO() dto.name = task.response.name dto.project = "" diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 40cdb1790..cf3128b61 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -18,6 +18,7 @@ import javax.crypto.spec.SecretKeySpec; import com.alibaba.fastjson.JSON; import com.alibaba.fastjson.JSONArray; import org.apache.commons.codec.binary.Base64; +import org.apache.commons.collections.CollectionUtils; import org.apache.http.HttpResponse; import org.apache.http.HttpStatus; import org.apache.http.client.HttpClient; @@ -270,13 +271,13 @@ public class ApiUtil { /** * 鑾峰彇浜哄憳浠诲姟鍒楄〃 */ - public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { + public static List listTaskByPerson(String userid, String startTime, String endTime, int limitStart, int limitNum) { List tasks = null; try { JSONObject body = new JSONObject(); body.put("limitStart", limitStart); body.put("limitNum", limitNum); - //body.put("userid", ""); + body.put("userid", userid); body.put("type", ""); body.put("timezone", ""); body.put("createtimeStart", startTime); @@ -303,6 +304,12 @@ public class ApiUtil { return tasks; } + /** + * 鑾峰彇浜哄憳浠诲姟鍒楄〃 + */ + public static List listTaskByPerson(String startTime, String endTime, int limitStart, int limitNum) { + return listTaskByPerson(null, startTime, endTime, limitStart, limitNum); + } /** * 鑾峰彇浠诲姟璇︽儏 * @@ -335,10 +342,26 @@ public class ApiUtil { public static void main(String[] args) { //List projects = ApiUtil.listProject(); - System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); - ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); - //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); + //System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); + //ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); + System.out.println(ApiUtil.getTaskDetail("e3f81b0381b44696b06fb6242a44902f")); //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); +// int limitStart = 0; +// List tasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50); +// List tempTasks = tasks; +// +// while (tasks != null && tempTasks != null && tempTasks.size() == 50) { +// limitStart += 50; +// tempTasks = ApiUtil.listTaskByPerson("055551305431136413","2021-03-01 00:00:00", "2021-04-30 23:59:59", limitStart, 50); +// if (CollectionUtils.isNotEmpty(tempTasks)) { +// tasks.addAll(tempTasks); +// } +// } +// for (Task task : tasks) { +// if ("绌嗚壇涔�".equals(task.getResponse().getName())) { +// System.out.println(task); +// } +// } } } From eca628115f9ac473baaaa694131cb0f17a3c2a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 11 May 2021 17:48:47 +0800 Subject: [PATCH 029/212] =?UTF-8?q?=E8=B0=83=E7=94=A8=E8=B6=85=E9=99=90?= =?UTF-8?q?=E5=90=8E=E7=9D=A1=E7=9C=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/statistics/ProjectExport.groovy | 5 ++++- .../shenjian/statistics/ScoreExport.groovy | 4 +++- .../shenjian/statistics/TaskExport.groovy | 4 +++- .../fangjia/script/statistics/ApiUtil.java | 20 ++++++++++++++----- 4 files changed, 25 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy index 2bd88e38b..0a243b841 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy @@ -22,6 +22,8 @@ class ProjectExport { export() } + + static void export() { List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; @@ -62,7 +64,8 @@ class ProjectExport { TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) if (taskDetail == null) { print("当前任务taskDetail为空:" + task) - continue; + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) if (taskDetail.getFlowConten() != null) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy index 81b26e748..58285190c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy @@ -61,7 +61,9 @@ public class ScoreExport { // 获取计划时间在当月的任务 TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) if (taskDetail == null) { - continue; + print("当前任务taskDetail为空:" + task) + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { boolean find = false diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy index 57e76d265..664b98e00 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy @@ -41,7 +41,9 @@ class TaskExport { // 获取计划时间在当月的任务 TaskDetail taskDetail = ApiUtil.getTaskDetail(task.getId()) if (taskDetail == null) { - continue; + print("当前任务taskDetail为空:" + task) + Thread.sleep(60 * 1000) + taskDetail = ApiUtil.getTaskDetail(task.getId()) } if ("穆良书" == task.getResponse().getName() && task.getId() == "e3f81b0381b44696b06fb6242a44902f") { print("穆良书") diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index cf3128b61..977bc2888 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -292,12 +292,17 @@ public class ApiUtil { param.put("corpid", corpId); param.put("body", body); JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskListByPerson", param); - if ((Integer)result.get("code") == 0) { + int code = (Integer)result.get("code"); + if (code == 0) { if ((Integer)JSONObject.parseObject(String.valueOf(result.get("data"))).get("count") > 0) { JSONArray jsonArray = JSONObject.parseObject(String.valueOf(result.get("data"))).getJSONArray("list"); tasks = jsonArray.toJavaList(Task.class); } - } + } else if (code == 90001){ + throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:listTaskByPerson"); + } else if (code == 90009) { + throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:listTaskByPerson"); + } } catch (Exception ex) { ex.printStackTrace(); } @@ -330,10 +335,15 @@ public class ApiUtil { param.put("corpid", corpId); param.put("body", body); JSONObject result = ApiUtil.post("https://api.antbim.net/ant_api/task/getTaskDetail", param); - if ((Integer)result.get("code") == 0) { + int code = (Integer)result.get("code"); + if (code == 0) { JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data"))); detail = obj.toJavaObject(TaskDetail.class); - } + } else if (code == 90001){ + throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:getTaskDetail"); + } else if (code == 90009) { + throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:getTaskDetail"); + } } catch (Exception e) { e.printStackTrace(); } @@ -344,7 +354,7 @@ public class ApiUtil { //List projects = ApiUtil.listProject(); //System.out.println(ApiUtil.listTaskByProject("e7caf92da7be4c529c8733cbb196173e", "2021-04-01", "2021-05-15", 0, 50)); //ApiUtil.listProjectLabel("e7caf92da7be4c529c8733cbb196173e"); - System.out.println(ApiUtil.getTaskDetail("e3f81b0381b44696b06fb6242a44902f")); + System.out.println(ApiUtil.getTaskDetail("be85282dc57d46e3a36fc0801107e605")); //System.out.println(ApiUtil.getTaskDetail("d08c7c2c852048aebc8aa59d48cfbd21")); //System.out.println(ApiUtil.listTaskByPerson("2021-04-01 00:00:00", "2021-04-30 23:59:59")); // int limitStart = 0; From 14de7390895b098f60d27e45f57037f5e232effb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 10:28:34 +0800 Subject: [PATCH 030/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=A9=E6=95=88?= =?UTF-8?q?=E6=97=B6=E9=97=B4=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/dingding/DingTalkUtils.groovy | 24 +++++++++++++++++++ .../ProjectExport.groovy | 10 ++++---- .../ScoreExport.groovy | 8 ++++--- .../TaskExport.groovy | 20 +++++++++------- .../shenjian/statistics/DingTalkUtils.groovy | 13 ---------- 5 files changed, 46 insertions(+), 29 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{statistics => dingding}/ProjectExport.groovy (95%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{statistics => dingding}/ScoreExport.groovy (96%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/{statistics => dingding}/TaskExport.groovy (90%) delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy new file mode 100644 index 000000000..5dda32688 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -0,0 +1,24 @@ +package com.fangjia.script.temp.shenjian.dingding + +import com.fangjia.script.util.DateUtils + + +/** + * 钉钉接口调用工具类 + * + * @author shenjian + * @since 2021/5/8 + */ +class DingTalkUtils { + + static main(String[] args) { + Date now = DateUtils.parseFromTightByPattern("2021-04-01", "yyyy-MM-dd") ; + + String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" + String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") + String endTime = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(now), "yyyy-MM-dd") + " 23:59:59" + ScoreExport.export(startTime, monthStartTime, endTime) + ProjectExport.export(monthStartTime, monthStartTime, endTime) + TaskExport.export(endTime, monthStartTime, endTime) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy similarity index 95% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy index 0a243b841..efd9a2e3b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy @@ -1,3 +1,5 @@ +package com.fangjia.script.temp.shenjian.dingding + import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel @@ -18,13 +20,11 @@ class ProjectExport { static final String monthStartTime = "2021-04-01 00:00:00" static final String endTime = "2021-04-30 23:59:59" - static void main(String[] args) { - export() + static main(args) { + export(startTime, monthStartTime, endTime) } - - - static void export() { + static export(def startTime, def monthStartTime, def endTime) { List> mergeData = new ArrayList<>() def sort = ["建行项目", "基础建设", "非农行项目", "农行项目"]; def result = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy similarity index 96% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy index 58285190c..511c72846 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy @@ -1,3 +1,5 @@ +package com.fangjia.script.temp.shenjian.dingding + import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel @@ -18,11 +20,11 @@ public class ScoreExport { static final String monthStartTime = "2021-04-01 00:00:00" static final String endTime = "2021-04-30 23:59:59" - static void main(String[] args) { - export() + static main(args) { + export(startTime, monthStartTime, endTime) } - static void export() { + static void export(def startTime, def monthStartTime, def endTime) { List> mergeData = new ArrayList<>() def sortedResult = [] sortedResult << ['项目类别', '工时', '所属项目', '项目经理'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy similarity index 90% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 664b98e00..632d6ecb9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -1,3 +1,5 @@ +package com.fangjia.script.temp.shenjian.dingding + import com.fangjia.script.statistics.ApiUtil import com.fangjia.script.statistics.Project import com.fangjia.script.statistics.ProjectLabel @@ -12,30 +14,32 @@ import org.apache.commons.collections.CollectionUtils * @author shenjian * @since 2021/5/10 */ -class TaskExport { +public class TaskExport { static final String time = "2021年4月1日-2021年4月30日"; static final String startTime = "2021-03-01 00:00:00" static final String monthStartTime = "2021-04-01 00:00:00" static final String endTime = "2021-04-30 23:59:59" - static void main(String[] args) { - export() + static main(args) { + export(startTime, monthStartTime, endTime) } - static void export() { + static void export(def startTime, def monthStartTime, def endTime) { List> mergeData = new ArrayList<>() List list = new ArrayList<>() def result = [] result << ['姓名', '所属项目', '参与任务', '任务绩效值', '投入时间(天)'] int limitStart = 0 - List tasks = new ArrayList<>() + List tasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) List tempTasks = tasks; - while (tempTasks.isEmpty() || tempTasks.size() == 50) { - tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + while (tasks != null && tempTasks != null && tempTasks.size() == 50) { limitStart += 50 - tasks.addAll(tempTasks) + tempTasks = ApiUtil.listTaskByPerson(startTime, endTime, limitStart, 50) + if (CollectionUtils.isNotEmpty(tempTasks)) { + tasks.addAll(tempTasks) + } } for (Task task: tasks) { // 获取计划时间在当月的任务 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy deleted file mode 100644 index 19efb9171..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/statistics/DingTalkUtils.groovy +++ /dev/null @@ -1,13 +0,0 @@ - -/** - * 钉钉接口调用工具类 - * - * @author shenjian - * @since 2021/5/8 - */ -class DingTalkUtils { - - static void main(String[] args) { - - } -} \ No newline at end of file From fed7c8763f270582f5872b003579ff40a9a1a69f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 10:56:56 +0800 Subject: [PATCH 031/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=B0=8F=E8=AE=A1?= =?UTF-8?q?=E5=AF=BC=E5=87=BAbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/dingding/TaskExport.groovy | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 632d6ecb9..a86e1a959 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -101,8 +101,14 @@ public class TaskExport { } def total = [] total[2] = "小计" - total[3] = map.get(result[result.size() - 1][0]).get("totalScore") - total[4] = map.get(result[result.size() - 1][0]).get("workTime") + if (map.get(result[result.size() - 1][0]) != null) { + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + } else { + total[3] = 0 + total[4] = 0 + print("无该信息" + result[result.size() - 1][0]) + } result << total for (Map info : map.values()) { From e20f9c22f1aad5c996753994ea40ea109a848ef6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 12 May 2021 10:56:56 +0800 Subject: [PATCH 032/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BC=A0=E5=8F=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/dingding/DingTalkUtils.groovy | 4 ++-- .../script/temp/shenjian/dingding/TaskExport.groovy | 10 ++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 5dda32688..00e7644b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -18,7 +18,7 @@ class DingTalkUtils { String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") String endTime = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(now), "yyyy-MM-dd") + " 23:59:59" ScoreExport.export(startTime, monthStartTime, endTime) - ProjectExport.export(monthStartTime, monthStartTime, endTime) - TaskExport.export(endTime, monthStartTime, endTime) + ProjectExport.export(startTime, monthStartTime, endTime) + TaskExport.export(startTime, monthStartTime, endTime) } } \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 632d6ecb9..a86e1a959 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -101,8 +101,14 @@ public class TaskExport { } def total = [] total[2] = "小计" - total[3] = map.get(result[result.size() - 1][0]).get("totalScore") - total[4] = map.get(result[result.size() - 1][0]).get("workTime") + if (map.get(result[result.size() - 1][0]) != null) { + total[3] = map.get(result[result.size() - 1][0]).get("totalScore") + total[4] = map.get(result[result.size() - 1][0]).get("workTime") + } else { + total[3] = 0 + total[4] = 0 + print("无该信息" + result[result.size() - 1][0]) + } result << total for (Map info : map.values()) { From 07e725fe6e4cafe792b20964368763b3109aa36b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 14:32:09 +0800 Subject: [PATCH 033/212] =?UTF-8?q?58=E7=A7=8D=E5=AD=90=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/crawl/City58.groovy | 26 +++++++---- .../script/temp/shenjian/crawl/Seed58.groovy | 44 +++++++++++++++++++ .../script/temp/shenjian/crawl/SeedBak.groovy | 33 ++++++++++++++ 3 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy index 78f033fd4..8b9abf395 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy @@ -252,23 +252,31 @@ class City58 { " }"; static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) static void main(String[] args) { JSONObject jsonObject = JSONObject.parseObject(cityList) Set provinceSet = jsonObject.keySet() List ret = new ArrayList<>() + + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + for (String province: provinceSet) { JSONObject provinceInfo = jsonObject[province] for (String city : provinceInfo.keySet()) { - def city_short = provinceInfo[city].split("\\|")[0] - DBObject dbObject = new BasicDBObject() - dbObject.put("province", province) - dbObject.put("city", city) - dbObject.put("city_short", city_short) - dbObject.put("c_date", new Date()) - dbObject.put("m_date", new Date()) - dbObject.put("root", "58.com") - ret.add(dbObject) + // 屏蔽县 + if (citySet.contains(city)) { + def city_short = provinceInfo[city].split("\\|")[0] + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", city_short) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "58.com") + ret.add(dbObject) + } } } cityConn.insert(ret) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy new file mode 100644 index 000000000..79bf00e9a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 58有效种子梳理 + * + * @author shenjian + * @since 2021/5/14 + */ +class Seaweed58 { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static void main(String[] args) { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + + DBObject query = new BasicDBObject() + query.put("root", "58.com") + + DBObject filter = new BasicDBObject() + filter.put("city", "1") + filter.put("_id", "1") + def delIds = [] + districtSeedsConn.find(query, filter).batchSize(20000).each {obj -> + if (!citySet.contains(obj.city)) { + delIds.add(obj._id) + if (delIds.size() == 10000) { + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + } + } + } + districtSeedsConn.remove(new BasicDBObject("_id", new BasicDBObject('$in', delIds))) + delIds = [] + print("58种子清理成功") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy new file mode 100644 index 000000000..56a389f28 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/SeedBak.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 种子数据备份 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeedBak { + + static DBCollection districtSeedsConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static DBCollection aimConn = MongoUtils.getCollectionByName("seeddb_dev", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + List list = new ArrayList<>() + + districtSeedsConn.find().batchSize(20000).each {obj -> + list.add(obj) + if (list.size() == 20000) { + aimConn.insert(list) + list.clear() + } + } + aimConn.insert(list) + list.clear() + println("种子备份成功") + } +} From 07f1398a92d7194c8c54f00f7461019094988d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 17:42:30 +0800 Subject: [PATCH 034/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E7=89=A9=E4=B8=9A=E7=AD=89?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy new file mode 100644 index 000000000..16e797245 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 重庆农商行 + * 开发商物业公司等小区信息导出 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeaweedExport { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static def map = new ConcurrentHashMap() + + static void main(String[] args) { + def result = new ConcurrentLinkedQueue() + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", name) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("developer", 1) + filter.put("estate_company", 1) + filter.put("cubage_percent", 1) // 容积率 + filter.put("virescence_percent", 1) // 绿化率 + filter.put("alias", 1) + filter.put("car_site_count", 1) + filter.put("all_area", 1) + filter.put("loudong_count", 1) + filter.put("household_count", 1) + filter.put("estate_charge", 1) + filter.put("open_date", 1) // 开盘时间 + filter.put("live_date", 1) // 交房时间 + + DbUtils.seaweed.find(query).batchSize(10).each { obj -> + def ret = [] + ret[0] = obj.developer + ret[1] = obj.estate_company + ret[2] = obj.cubage_percent + ret[3] = obj.virescence_percent + + if (obj.alias) { + // 小区可能存在多个别名 + def temp = [] + for (def a : obj.alias) { + temp.add(a) + } + ret[4] = temp.join(",") + } + ret[5] = obj.car_site_count + ret[6] = "" + ret[7] = "" + ret[8] = obj.all_area + ret[9] = obj.loudong_count + ret[10] = obj.household_count + ret[11] = "" + ret[12] = obj.estate_charge + ret[13] = obj.open_date + ret[14] = obj.live_date + result.add(ret) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret = [] + for (def r : result) { + ret << r + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商物业公司.xlsx", ret, "项目统计"); + executorService.shutdown() + } +} From 30d5583a52b1ca74ad2d4be21d677d36874ed17a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 14 May 2021 17:42:30 +0800 Subject: [PATCH 035/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=BC=80=E5=8F=91=E5=95=86=E7=89=A9=E4=B8=9A=E7=AD=89?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AroundExport.groovy | 18 +++ .../temp/shenjian/cqrcb/Constants.groovy | 10 ++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 126 ++++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 60 +++++++++ 4 files changed, 214 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy new file mode 100644 index 000000000..40851320a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -0,0 +1,18 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + +/** + * 小区周围信息导出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AroundExport { + + def static types = "" + + static void main(String[] args) { + GaoDeUtils.searchNearByPoi() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy new file mode 100644 index 000000000..bfafad05d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/Constants.groovy @@ -0,0 +1,10 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +/** + * @author shenjian + * @since 2021/5/15 + */ +class Constants { + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy new file mode 100644 index 000000000..445b953f6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -0,0 +1,126 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 重庆农商行 + * 开发商物业公司等小区信息导出 + * + * @author shenjian + * @since 2021/5/14 + */ +class SeaweedExport { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static def map = new ConcurrentHashMap() + + static void main(String[] args) { + def resultMap = new ConcurrentHashMap() + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", name) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("developer", 1) + filter.put("estate_company", 1) + filter.put("cubage_percent", 1) // 容积率 + filter.put("virescence_percent", 1) // 绿化率 + filter.put("alias", 1) + filter.put("car_site_count", 1) + filter.put("all_area", 1) + filter.put("loudong_count", 1) + filter.put("household_count", 1) + filter.put("estate_charge", 1) + filter.put("open_date", 1) // 开盘时间 + filter.put("live_date", 1) // 交房时间 + filter.put("lng", 1) + filter.put("lat", 1) + + def ret = [] + DbUtils.seaweed.find(query).limit(1).each { obj -> + ret[0] = obj.developer + ret[1] = obj.estate_company + ret[2] = obj.cubage_percent + ret[3] = obj.virescence_percent + + if (obj.alias) { + // 小区可能存在多个别名 + def temp = [] + for (def a : obj.alias) { + temp.add(a) + } + ret[4] = temp.join(",") + } + ret[5] = obj.car_site_count + ret[6] = "" + ret[7] = "" + ret[8] = obj.all_area + ret[9] = obj.loudong_count + ret[10] = obj.household_count + ret[11] = "" + ret[12] = obj.estate_charge + ret[13] = obj.open_date + ret[14] = obj.live_date + ret[15] = obj.lng + ret[16] = obj.lat + } + ret[17] = name + resultMap.put(district_code + name, ret) + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret = [] + ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"] + for (List district : data) { + def district_code = district.get(2) + def name = district.get(3) + ret << resultMap.get(district_code + name) + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商物业公司.xlsx", ret, "项目统计"); + executorService.shutdown() + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..155740221 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,60 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.HttpRequestUtil +import net.sf.json.JSONObject + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde" + + /** + * + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, String radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = radius + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else { + break + } + } + return result + } +} From 0793de69327edd760ff6bd5d8b95c7fca18d3c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 15 May 2021 11:52:30 +0800 Subject: [PATCH 036/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=B0=8F=E5=8C=BA=E5=91=A8=E5=9B=B4=E5=BB=BA=E7=AD=91?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AroundExport.groovy | 50 ++++++++++++++++++- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 1 - 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy index 40851320a..9c53f103f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -1,6 +1,18 @@ package com.fangjia.script.temp.shenjian.cqrcb +import cn.emay.mina.util.ConcurrentHashSet +import com.fangjia.script.coredata.util.CollectionUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger /** * 小区周围信息导出 @@ -10,9 +22,43 @@ import com.fangjia.script.temp.shenjian.util.GaoDeUtils */ class AroundExport { - def static types = "" + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - GaoDeUtils.searchNearByPoi() + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + def city = district.get(1) + def name = district.get(3) + if (district.get(20) != null && district.get(21) != null) { + boolean exist = conn.count(new BasicDBObject("district_name", name)) > 0 ? true : false + if (!exist) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("district_name", name) + obj.put("lng", lng) + obj.put("lat", lat) + } + conn.insert(arounds) + } + } + } + println("当前处理小区" + name + " " + count.incrementAndGet()) + } + }) + } + executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 445b953f6..8d48e0319 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -8,7 +8,6 @@ import com.mongodb.DBCollection import com.mongodb.DBObject import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors From 6a679dc5fcd6f6db6adb70b1acf34c21a147b0a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sat, 15 May 2021 14:04:53 +0800 Subject: [PATCH 037/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=B0=8F=E5=8C=BA=E5=91=A8=E5=9B=B4=E5=BB=BA=E7=AD=91?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AllExport.groovy | 74 ++++++++ .../temp/shenjian/cqrcb/AroundExport.groovy | 55 ++++-- .../cqrcb/AroundExportByNewWay.groovy | 174 ++++++++++++++++++ .../shenjian/cqrcb/AroundFormatExport.groovy | 161 ++++++++++++++++ .../shenjian/cqrcb/MarkUnNeedExport.groovy | 32 ++++ .../shenjian/cqrcb/MoveArroundTemp.groovy | 64 +++++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 47 ++++- .../temp/shenjian/school/DistanceDeal.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 3 +- 9 files changed, 583 insertions(+), 31 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy new file mode 100644 index 000000000..99bf24d62 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy @@ -0,0 +1,74 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 重庆所有小区周边导出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AllExport { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("cat", "district") + query.put("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + AtomicInteger count = new AtomicInteger(0) + DBUtils.seaweed.find(query, filter).batchSize(10000).each {district -> + executorService.execute(new Runnable() { + @Override + void run() { + def city = "重庆" + def name = district.name + def region = district.region + boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false + if (!exist) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lng / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.lat / 10000000000d) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("matched_city", city) + obj.put("matched_region", district.region) + obj.put("matched_name", district.name) + } + conn.insert(arounds) + } + println("当前处理小区" + name + " " + count.incrementAndGet()) + } else { + println("小区已存在") + } + } + }) + } + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy index 9c53f103f..fd3450f87 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.temp.shenjian.cqrcb import cn.emay.mina.util.ConcurrentHashSet import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoUtils @@ -27,38 +28,54 @@ class AroundExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" static def radius = "3000" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + //static ExecutorService executorService = Executors.newFixedThreadPool(30) + // 高德地图中不存在周边的数据 + // 重庆 江北 阳光丽苑 + //重庆 江北 金田花苑 + //重庆 巴南 巴滨一号璀璨滨江 + //重庆 丰都 茅林沟移民新村 + //重庆 江津 福城北苑 + //重庆 巫山 摩天岭壹号 static void main(String[] args) { - List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 0) AtomicInteger count = new AtomicInteger(0) for (List districtT : data) { final List district = districtT - executorService.execute(new Runnable() { - @Override - void run() { - def city = district.get(1) - def name = district.get(3) - if (district.get(20) != null && district.get(21) != null) { - boolean exist = conn.count(new BasicDBObject("district_name", name)) > 0 ? true : false - if (!exist) { - def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) - def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) +// executorService.execute(new Runnable() { +// @Override +// void run() { + def city = district.get(0) + def region = district.get(1) + def name = district.get(2) + boolean exist = conn.count(new BasicDBObject("matched_city": city, "matched_region": region,"matched_name": name)) > 0 ? true : false + if (!exist) { + DBObject query = new BasicDBObject() + query.put("city", city) + query.put("region", region) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", name) + DBUtils.seaweed.find(query, new BasicDBObject("lat", 1).append("lng", 1)).limit(1).each {objT -> + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lng / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, objT.lat / 10000000000d) List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") if (CollectionUtils.isNotEmpty(arounds)) { for (BasicDBObject obj : arounds) { - obj.put("district_name", name) - obj.put("lng", lng) - obj.put("lat", lat) + obj.put("matched_city", city) + obj.put("matched_region", region) + obj.put("matched_name", name) } conn.insert(arounds) } + println("当前处理小区" + name + " " + count.incrementAndGet()) } + } else { + println("小区已存在") } - println("当前处理小区" + name + " " + count.incrementAndGet()) - } - }) +// } +// }) } - executorService.shutdown() + //executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy new file mode 100644 index 000000000..7053f1db6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -0,0 +1,174 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 周边数据导出新方式 + * + * @author shenjian + * @since 2021/5/16 + */ +class AroundExportByNewWay { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + static def resultMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def id = district.get(0) + def district_code = district.get(2) + if (district_code == 'B100203') { + println(district_code) + } + def name = district.get(3) + List result = [] + if (district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) + DBObject q = new BasicDBObject() + q.put('status', 1) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + Map map = new ConcurrentHashMap<>(); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 3000) { + dealSpecial(obj, map, id, distance, result) + } + } + for (String key : map.keySet()) { + int distance = map.get(key).get("distance") + int count1 = map.get(key).get("count") + def type = map.get(key).get("type") + def ret = [] + ret[0] = id + ret[1] = key + ret[2] = type + ret[3] = (distance / count1).toInteger() + result << ret + } + } + resultMap.put(district_code + name, result) + println("当前处理" + count.incrementAndGet() ) + } catch (Exception ex) { + + } finally { + latch.countDown() + } + } + }) + }; + latch.await() + def ret = [] + ret << ['IDB', '名称','类型', '距离'] + for (List district : data) { + def district_code = district.get(2) + def name = district.get(3) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + ret.addAll(resultMap.get(district_code + name)) + } + } + println(ret.size()) + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + executorService.shutdown() + + } + + static dealSpecial(DBObject obj, def map, def id, double distance, def result) { + def name = obj.name + if (name.indexOf("/") != -1) { + return false + } + if (name == "1号楼") { + return false + } + if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { + if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { + if (name.indexOf("分院") == -1) { + return false + } + } + if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { + return false + } + if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { + if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 + && !name.endsWith("医院")) { + return false + } + } + if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { + return false + } + if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { + return false + } + if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { + if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { + return false + } + } + if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + } + // 中梁山地铁站1A口 + if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { + def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) + def key = nameTemp + Map child = new HashMap() + if (map.get(key) != null) { + distance + map.get(key).get("distance") + int count1 = map.get(key).get("count") + child.put("distance", distance) + child.put("count", count1) + child.put("type", obj.type) + map.put(key, child) + } else { + child.put("distance", distance) + child.put("type", obj.type) + child.put("count", 1) + map.put(key, child) + } + } else { + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = obj.type + ret[3] = distance.toInteger() + result << ret + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy new file mode 100644 index 000000000..b445e881c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -0,0 +1,161 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import cn.emay.mina.util.ConcurrentHashSet +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区周围信息格式化输出 + * + * @author shenjian + * @since 2021/5/15 + */ +class AroundFormatExport { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static def resultMap = new ConcurrentHashMap() + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + AtomicInteger count = new AtomicInteger(0) + CountDownLatch latch = new CountDownLatch(data.size()) + for (List districtT : data) { + final List district = districtT + executorService.execute(new Runnable() { + @Override + void run() { + try { + def district_code = district.get(2) + def name = district.get(3) + def temp = [] + if (district.get(20) != null && district.get(21) != null) { + def matched_city = district.get(22) + def matched_region = district.get(23) + def matched_name = district.get(24) + + DBObject query = new BasicDBObject() + query.put("matched_city", matched_city) + query.put("matched_region", matched_region) + query.put("matched_name", matched_name) + List ret = new ArrayList() + Map map = new ConcurrentHashMap<>(); + conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> + dealSpecial(obj, map, ret) + } + for (String key : map.keySet()) { + int distance = map.get(key).get("distance") + int count1 = map.get(key).get("count") + ret.add(key + ":" + (distance / count1) + "米") + } + + temp[7] = matched_city + temp[8] = matched_region + temp[9] = matched_name + if (ret.size() > 0) { + def str = ret.join(",") + temp[5] = str + println(str) + } + + } + resultMap.put(district_code + name, temp) + println("当前处理" + count.incrementAndGet() ) + } catch (Exception ex) { + + } finally { + latch.countDown() + } + } + }) + } + latch.await() + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] + for (List district : data) { + def district_code = district.get(2) + def name = district.get(3) + ret << resultMap.get(district_code + name) + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + executorService.shutdown() + } + + static dealSpecial(DBObject obj, def map, def ret) { + def name = obj.name + if (name.indexOf("/") != -1) { + return false + } + if (name == "1号楼") { + return false + } + if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { + if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { + if (name.indexOf("分院") == -1) { + return false + } + } + if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { + return false + } + if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { + if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 + && !name.endsWith("医院")) { + return false + } + } + if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { + return false + } + if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { + return false + } + if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { + if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { + return false + } + } + if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { + if (name.indexOf("校区") == -1) { + return false + } + } + } + // 中梁山地铁站1A口 + if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { + def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) + def key = nameTemp + Map child = new HashMap() + if (map.get(key) != null) { + int distance = Double.valueOf(obj.distance) + map.get(key).get("distance") + int count1 = map.get(key).get("count") + child.put("distance", distance) + child.put("count", count1) + map.put(key, child) + } else { + child.put("distance", Double.valueOf(obj.distance)) + child.put("count", 1) + map.put(key, child) + } + } else { + ret.add(obj.name + ":" + obj.distance + "米") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy new file mode 100644 index 000000000..76226f770 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators + +/** + * 标记不需要导出的数据 + * + * @author shenjian + * @since 2021/5/16 + */ +class MarkUnNeedExport { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边补充.xlsx", 1) + for (List districtT : data) { + final List district = districtT + if (district.size() > 1) { + def type = district.get(0) + def status = district.get(1) + if (status == 1) { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1))) + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy new file mode 100644 index 000000000..e17e12840 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy @@ -0,0 +1,64 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * POI迁移 + * + * @author shenjian + * @since 2021/5/16 + */ +class MoveArroundTemp { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + List list = Collections.synchronizedList(new ArrayList()) + List ids = conn.distinct("id") + CountDownLatch latch = new CountDownLatch(ids.size()) + for (String idT : ids) { + final String id = idT + executorService.submit(new Runnable() { + @Override + void run() { + try { + DBObject dbObject = new BasicDBObject() + conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> + dbObject.put("_id", id) + dbObject.put("name", obj.name) + dbObject.put("type", obj.type) + dbObject.put("typecode", obj.typecode) + dbObject.put("address", obj.address) + dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) + dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) + dbObject.put("tel", obj.tel) + dbObject.put("province", obj.pname) + dbObject.put("city", obj.cityname) + dbObject.put("region", obj.adname) + } + list.add(dbObject) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + println("当前记录数" + list.size()) + } + } + }) + }; + latch.await(); + conn43.insert(list); + list.clear(); + println("同步清理完成"); + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 8d48e0319..feb17ea7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.cqrcb +import com.fangjia.core.util.DateUtil import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -38,7 +39,11 @@ class SeaweedExport { def city = district.get(1) def district_code = district.get(2) def name = district.get(3) + def matchedCity = district.get(22) + def matchedRegion = district.get(23) + def matchedName = district.get(24) def region; + if (map.get("district_code") != null) { region = map.get("district_code") } else { @@ -47,12 +52,17 @@ class SeaweedExport { map.put("district_code", region) } } - + if (matchedRegion == null) { + matchedRegion = region + matchedName = name + } + def ret = [] DBObject query = new BasicDBObject() query.put("city", city) + query.put("region", matchedRegion) query.put("cat", "district") query.put("visible", 0) - query.put("name", name) + query.put("name", matchedName) DBObject filter = new BasicDBObject() filter.put("_id", 1) @@ -71,12 +81,11 @@ class SeaweedExport { filter.put("lng", 1) filter.put("lat", 1) - def ret = [] DbUtils.seaweed.find(query).limit(1).each { obj -> ret[0] = obj.developer ret[1] = obj.estate_company ret[2] = obj.cubage_percent - ret[3] = obj.virescence_percent + ret[3] = virescencePercentValidator(obj.virescence_percent) if (obj.alias) { // 小区可能存在多个别名 @@ -94,7 +103,7 @@ class SeaweedExport { ret[10] = obj.household_count ret[11] = "" ret[12] = obj.estate_charge - ret[13] = obj.open_date + ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date) ret[14] = obj.live_date ret[15] = obj.lng ret[16] = obj.lat @@ -108,9 +117,9 @@ class SeaweedExport { } } }) - } - latch.await() - def ret = [] + }; + latch.await(); + def ret = []; ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"] for (List district : data) { def district_code = district.get(2) @@ -121,5 +130,27 @@ class SeaweedExport { executorService.shutdown() } + /** + * string + * 绿化率 仅限数字,包括(小数)数字范围:10-100 + * @param virescence_percent + * @return + */ + private static def virescencePercentValidator(def virescence_percent){ + if(virescence_percent=="null"){ + return ""; + } + def value = com.fangjia.script.datastore.DistrictFieldNumberic.checkValue(virescence_percent); + if(value){ + if(value>=10 && value<=100){ + return value; + } else if (value > 0 && value < 1) { + return value * 100 + } else{ + return ""; + } + } + return ""; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy index d19524f07..3037ee807 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -33,8 +33,8 @@ class DistanceDeal { // updateField.put("school_lat", "31.330963") // collection.update(new BasicDBObject("school_url", "https://hf.esf.fang.com/school/10483.htm"), new BasicDBObject('$set', updateField), false, true) -// double length = Distance.GetDistance(); -// print(length) + double length = Distance.GetDistance(); + print(length) // DBObject updateField = new BasicDBObject() // updateField.put("school_lng", 120.68764) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 155740221..eef249210 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,7 +1,6 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.core.util.DataUtil -import com.fangjia.script.util.GeoUtils import com.fangjia.script.util.HttpRequestUtil import net.sf.json.JSONObject @@ -13,7 +12,7 @@ import net.sf.json.JSONObject */ class GaoDeUtils { - def static GAO_DE_AK = "4ccfe3c898a9489884bd2bcb1ef6dcde" + def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988" /** * From 37580693794fb056fae5ea093e94bcd537ae5def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 16 May 2021 16:06:51 +0800 Subject: [PATCH 038/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=B0=8F=E5=8C=BA=E5=91=A8=E5=9B=B4=E5=BB=BA=E7=AD=91?= =?UTF-8?q?=E4=BF=A1=E6=81=AF=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 22 +++++++++++++++---- .../shenjian/cqrcb/MarkUnNeedExport.groovy | 11 ++++++++-- .../temp/shenjian/util/GaoDeUtils.groovy | 16 ++++++++++++++ 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 7053f1db6..94e4b9342 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -1,13 +1,16 @@ package com.fangjia.script.temp.shenjian.cqrcb +import com.alibaba.excel.EasyExcel import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -27,6 +30,12 @@ class AroundExportByNewWay { static def resultMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) + static Map map = new HashedMap() + + static { + map.put("") + } + static void main(String[] args) { List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) AtomicInteger count = new AtomicInteger(0) @@ -51,11 +60,11 @@ class AroundExportByNewWay { // 周围半径为3千米 def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) DBObject q = new BasicDBObject() - q.put('status', 1) + q.put('status', 0) q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); Map map = new ConcurrentHashMap<>(); - poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) if (distance <= 3000) { dealSpecial(obj, map, id, distance, result) @@ -84,8 +93,8 @@ class AroundExportByNewWay { }) }; latch.await() + def headArr = ['IDB', '名称','类型', '距离'] def ret = [] - ret << ['IDB', '名称','类型', '距离'] for (List district : data) { def district_code = district.get(2) def name = district.get(3) @@ -94,7 +103,9 @@ class AroundExportByNewWay { } } println(ret.size()) - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + FileUtils.batchCreateFile(ret, "F:\\数据\\重庆农商行\\周边配套.dat", headArr, "utf-8") +// EasyExcel.write("F:\\数据\\重庆农商行\\周边配套.xlsx").head(head).sheet("成交信息").doWrite(ret); executorService.shutdown() } @@ -143,6 +154,9 @@ class AroundExportByNewWay { return false } } + if (name.indexOf("(出入口)") != -1) { + return false + } } // 中梁山地铁站1A口 if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy index 76226f770..c9eb96812 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy @@ -4,7 +4,9 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import com.mongodb.QueryOperators +import sun.swing.BakedArrayList /** * 标记不需要导出的数据 @@ -23,8 +25,13 @@ class MarkUnNeedExport { if (district.size() > 1) { def type = district.get(0) def status = district.get(1) - if (status == 1) { - poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1))) + if (type == '交通设施服务;公交车站;公交车站相关') { + println(type) + } + if (status == 1.0) { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 0)), false, true) + } else { + poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index eef249210..721ab6958 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -4,6 +4,8 @@ import com.fangjia.core.util.DataUtil import com.fangjia.script.util.HttpRequestUtil import net.sf.json.JSONObject +import java.util.concurrent.ConcurrentHashMap + /** * 高德工具类 * @@ -14,6 +16,20 @@ class GaoDeUtils { def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988" + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + /** * * @param city From d0744e2df5258c380241f9a0a15797f74dd6fb1b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 16 May 2021 19:29:35 +0800 Subject: [PATCH 039/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=A7=84=E5=88=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 9 +-- .../shenjian/cqrcb/MarkUnNeedExport.groovy | 3 - .../script/temp/shenjian/cqrcb/POIRule.groovy | 61 +++++++++++++++++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 5 +- .../temp/shenjian/util/GaoDeUtils.groovy | 14 ++++- 5 files changed, 81 insertions(+), 11 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 94e4b9342..564da168d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -30,12 +30,6 @@ class AroundExportByNewWay { static def resultMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) - static Map map = new HashedMap() - - static { - map.put("") - } - static void main(String[] args) { List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) AtomicInteger count = new AtomicInteger(0) @@ -119,6 +113,9 @@ class AroundExportByNewWay { return false } if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { + if (name.indexOf("栋") != -1) { + return false + } if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { if (name.indexOf("分院") == -1) { return false diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy index c9eb96812..9ce164947 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MarkUnNeedExport.groovy @@ -25,9 +25,6 @@ class MarkUnNeedExport { if (district.size() > 1) { def type = district.get(0) def status = district.get(1) - if (type == '交通设施服务;公交车站;公交车站相关') { - println(type) - } if (status == 1.0) { poiConn.update(new BasicDBObject("type", type), new BasicDBObject('$set', new BasicDBObject("status", 0)), false, true) } else { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy new file mode 100644 index 000000000..ca637b48e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.regex.Pattern + +/** + * POI清理规则 + * + * @author shenjian + * @since 2021/5/16 + */ +class POIRule { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + // 小学校转为学校 + poiConn.find(new BasicDBObject("typecode", "141203").append("status", 0)).batchSize(1000).each {obj -> + if (obj.name.indexOf('小学校') != -1 ) { + def id = obj._id + def region = obj.region + def name = obj.name.replace("小学校", '小学') + if (poiConn.count(new BasicDBObject("region": region, "name": name)) > 0) { + poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 1)), false, true) + } else { + poiConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("name", name)), false, true) + } + } + } + } + + static BasicDBObject getLikeStr(String findStr){ + Pattern pattern = Pattern.compile('^.*' + findStr + '.*$', Pattern.CASE_INSENSITIVE); + return new BasicDBObject('$regex', pattern); + } + +// 更新规则 +// db.getCollection('gaode_address_poi').update({'name': /栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /购物服务;商场;免税品店/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;口腔医院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;综合医院;卫生院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[学]+楼/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[教医光]+学中心/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;学校/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[1234]期/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /住院部/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /\//, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index feb17ea7f..69638ec6f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -85,6 +85,9 @@ class SeaweedExport { ret[0] = obj.developer ret[1] = obj.estate_company ret[2] = obj.cubage_percent + if (obj.developer == '重庆汇凯置业有限公司') { + println('重庆汇凯置业有限公司') + } ret[3] = virescencePercentValidator(obj.virescence_percent) if (obj.alias) { @@ -145,7 +148,7 @@ class SeaweedExport { if(value>=10 && value<=100){ return value; } else if (value > 0 && value < 1) { - return value * 100 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value) * 100) } else{ return ""; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 721ab6958..1ccbf1713 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -19,7 +19,19 @@ class GaoDeUtils { def static codeNameMap = new HashMap(){{ put("150500", "地铁"); put("1507", "公交"); - put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") put("141200", "学校") put("141201", "学校") put("141202", "学校") From 65b8edb112b14914dbfa8d78359f4cd94308be56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 10:17:12 +0800 Subject: [PATCH 040/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E8=BD=A6=E4=BD=8D?= =?UTF-8?q?=E6=AF=94=E5=8F=8A=E7=BB=BF=E5=8C=96=E7=8E=87=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/cqrcb/POIRule.groovy | 5 +++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 19 ++++++++++++++----- 2 files changed, 19 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index ca637b48e..e6ab78d3d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.cqrcb +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -18,6 +19,10 @@ class POIRule { static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static void main(String[] args) { + + } + + static void changeSchool() { // 小学校转为学校 poiConn.find(new BasicDBObject("typecode", "141203").append("status", 0)).batchSize(1000).each {obj -> if (obj.name.indexOf('小学校') != -1 ) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 69638ec6f..e5e264415 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.temp.shenjian.cqrcb import com.fangjia.core.util.DateUtil +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -85,9 +87,6 @@ class SeaweedExport { ret[0] = obj.developer ret[1] = obj.estate_company ret[2] = obj.cubage_percent - if (obj.developer == '重庆汇凯置业有限公司') { - println('重庆汇凯置业有限公司') - } ret[3] = virescencePercentValidator(obj.virescence_percent) if (obj.alias) { @@ -99,7 +98,17 @@ class SeaweedExport { ret[4] = temp.join(",") } ret[5] = obj.car_site_count - ret[6] = "" + if (com.fangjia.script.datastore.FieldValidatorUtils.householdCountValidator(obj.household_count) + && FieldValidatorUtils.carSiteValidator(obj.car_site_count)) { + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) + } + ret[6] = "1:" + val + } ret[7] = "" ret[8] = obj.all_area ret[9] = obj.loudong_count @@ -146,7 +155,7 @@ class SeaweedExport { def value = com.fangjia.script.datastore.DistrictFieldNumberic.checkValue(virescence_percent); if(value){ if(value>=10 && value<=100){ - return value; + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value)) } else if (value > 0 && value < 1) { return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value) * 100) } else{ From 84260e7d46687a69ffcbd3a29ecf548e21657941 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 10:42:29 +0800 Subject: [PATCH 041/212] =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 155 +++++++----------- .../temp/shenjian/cqrcb/DealUnMatched.groovy | 106 ++++++++++++ .../temp/shenjian/cqrcb/ImportSchool.groovy | 53 ++++++ .../shenjian/cqrcb/MoveArroundTemp.groovy | 81 +++++++-- .../script/temp/shenjian/cqrcb/POIRule.groovy | 8 +- .../script/temp/shenjian/cqrcb/SavePoi.groovy | 55 +++++++ .../temp/shenjian/school/DistanceDeal.groovy | 13 +- .../temp/shenjian/util/GaoDeUtils.groovy | 19 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 32 ++++ 9 files changed, 398 insertions(+), 124 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 564da168d..5147a5b04 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -28,6 +28,7 @@ class AroundExportByNewWay { static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -42,9 +43,6 @@ class AroundExportByNewWay { try { def id = district.get(0) def district_code = district.get(2) - if (district_code == 'B100203') { - println(district_code) - } def name = district.get(3) List result = [] if (district.get(20) != null && district.get(21) != null) { @@ -57,24 +55,28 @@ class AroundExportByNewWay { q.put('status', 0) q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); - Map map = new ConcurrentHashMap<>(); poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) if (distance <= 3000) { - dealSpecial(obj, map, id, distance, result) + if (distance == 0) { + distance = 50 + } + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = formatType(obj.typecode) + ret[3] = distance.toInteger() + result << ret } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - def type = map.get(key).get("type") - def ret = [] - ret[0] = id - ret[1] = key - ret[2] = type - ret[3] = (distance / count1).toInteger() - result << ret - } + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) @@ -86,100 +88,63 @@ class AroundExportByNewWay { } }) }; - latch.await() + latch.await(); def headArr = ['IDB', '名称','类型', '距离'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + Map map = new HashedMap() + Integer key = 1; def ret = [] for (List district : data) { def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { ret.addAll(resultMap.get(district_code + name)) + if (ret.size() > 600000) { + map.put(key++, ret) + ret = [] + } } } - println(ret.size()) + map.put(key++, ret) //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); - FileUtils.batchCreateFile(ret, "F:\\数据\\重庆农商行\\周边配套.dat", headArr, "utf-8") -// EasyExcel.write("F:\\数据\\重庆农商行\\周边配套.xlsx").head(head).sheet("成交信息").doWrite(ret); + for (Integer k : map.keySet()) { + FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + //EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + //printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def id, double distance, def result) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("栋") != -1) { - return false - } - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false - } - } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("(出入口)") != -1) { - return false - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - distance + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - child.put("type", obj.type) - map.put(key, child) - } else { - child.put("distance", distance) - child.put("type", obj.type) - child.put("count", 1) - map.put(key, child) - } - } else { - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = obj.type - ret[3] = distance.toInteger() - result << ret + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String type) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy new file mode 100644 index 000000000..48be458a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +/** + * 处理未匹配的小区 + * + * @author shenjian + * @since 2021/5/17 + */ +class DealUnMatched { + + static void main(String[] args) { + // getLatLngByName() + // getLatLngByNumber() + //findExceptionDistance() + + // B114862 + def lng = 107.736591 + def lat = 29.872502 + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 500) + DBObject q = new BasicDBObject() + q.put('city', '重庆') + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + DBUtils.seaweed.find(q).batchSize(100).each {obj -> + println(obj) + } + } + + static void findExceptionDistance() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + for (List district : data) { + if (district.size() > 12) { + def lng = java.lang.Double.valueOf(district.get(8)) + def lat = java.lang.Double.valueOf(district.get(9)) + def lng1 = java.lang.Double.valueOf(district.get(11)) + def lat1 = java.lang.Double.valueOf(district.get(12)) + double distance = GaoDeUtils.getDistance(lat, lng, lat1, lng1) + if (distance > 500) { + println("距离异常" + district) + district.add(distance) + district.add("距离异常") + } + } + } + data.add(0, new ArrayList()) + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "周边配套") + } + + static void getLatLngByNumber() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",1) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + + //def region = SeaweedUtils.getRegionByCode(district_code) + if (district.size() <= 5) { + if (number != null && number.size() > 5) { + def map = GaoDeUtils.parserByGaode(city, number) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过楼栋得出") + } + } else { + println(district_code + " " + name + " " + number) + } + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配1") + } + + static void getLatLngByName() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", 0) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + //def region = SeaweedUtils.getRegionByCode(district_code) + + def map = GaoDeUtils.parserByGaode(city, name) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过小区得出") + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy new file mode 100644 index 000000000..b8f0452ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy @@ -0,0 +1,53 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 导入中学小学信息 + * + * @author shenjian + * @since 2021/5/17 + */ +class ImportSchool { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1) + List result = new ArrayList(); + for (List school : data) { + try { + def city = school.get(0) + def region = school.get(1) + def name = school.get(2) + def address = school.get(4) + def type = school.get(6) + def lng = school.get(9) + def lat = school.get(10) + + DBObject dbObject = new BasicDBObject() + dbObject.put("province", "重庆") + dbObject.put("city", city) + dbObject.put("region", region) + dbObject.put("name", name) + dbObject.put("address", address) + dbObject.put("type", type) + dbObject.put("typecode", "141200") + dbObject.put("lng", lng) + dbObject.put("lat", lat) + dbObject.put("status", 0) + dbObject.put("flag", "excel导入") + result.add(dbObject) + } catch(Exception ex) { + println(school) + ex.printStackTrace() + } + } + + poiConn.insert(result) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy index e17e12840..c641a4fc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy @@ -1,13 +1,17 @@ package com.fangjia.script.temp.shenjian.cqrcb +import cn.emay.mina.util.ConcurrentHashSet import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger /** * POI迁移 @@ -22,18 +26,20 @@ class MoveArroundTemp { static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - List list = Collections.synchronizedList(new ArrayList()) - List ids = conn.distinct("id") - CountDownLatch latch = new CountDownLatch(ids.size()) - for (String idT : ids) { - final String id = idT + Map map = new ConcurrentHashMap() + AtomicInteger count = new AtomicInteger() + int size = conn.count(new BasicDBObject("status", 1)) + println(size) + CountDownLatch latch = new CountDownLatch(size) + conn.find(new BasicDBObject("status", 1)).batchSize(10000).each { objT -> + final def obj = objT executorService.submit(new Runnable() { @Override void run() { try { - DBObject dbObject = new BasicDBObject() - conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> - dbObject.put("_id", id) + if (map.get(obj.id) == null && conn43.count(new BasicDBObject("_id", obj.id)) <= 0) { + DBObject dbObject = new BasicDBObject() + dbObject.put("_id", obj.id) dbObject.put("name", obj.name) dbObject.put("type", obj.type) dbObject.put("typecode", obj.typecode) @@ -44,21 +50,66 @@ class MoveArroundTemp { dbObject.put("province", obj.pname) dbObject.put("city", obj.cityname) dbObject.put("region", obj.adname) + dbObject.put("status", 2) + dbObject.put("flag", "待清洗") + map.put(obj.id, dbObject) } - list.add(dbObject) } catch(Exception ex) { ex.printStackTrace() } finally { latch.countDown() - println("当前记录数" + list.size()) + println(count.incrementAndGet()) } } }) - }; - latch.await(); - conn43.insert(list); - list.clear(); - println("同步清理完成"); + } + latch.await() + List ret = new ArrayList() + for (String key : map.keySet()) { + ret << map.get(key) + } + conn43.insert(ret) executorService.shutdown() } + + static origin() { +// List list = Collections.synchronizedList(new ArrayList()) +// List ids = conn.distinct("id") +// CountDownLatch latch = new CountDownLatch(ids.size()) +// for (String idT : ids) { +// final String id = idT +// executorService.submit(new Runnable() { +// @Override +// void run() { +// try { +// DBObject dbObject = new BasicDBObject() +// conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> +// dbObject.put("_id", id) +// dbObject.put("name", obj.name) +// dbObject.put("type", obj.type) +// dbObject.put("typecode", obj.typecode) +// dbObject.put("address", obj.address) +// dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) +// dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) +// dbObject.put("tel", obj.tel) +// dbObject.put("province", obj.pname) +// dbObject.put("city", obj.cityname) +// dbObject.put("region", obj.adname) +// } +// list.add(dbObject) +// } catch(Exception ex) { +// ex.printStackTrace() +// } finally { +// latch.countDown() +// println("当前记录数" + list.size()) +// } +// } +// }) +// }; +// latch.await(); +// conn43.insert(list); +// list.clear(); +// println("同步清理完成"); +// executorService.shutdown() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index e6ab78d3d..88033d0b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -44,10 +44,11 @@ class POIRule { } // 更新规则 -// db.getCollection('gaode_address_poi').update({'name': /栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); +// db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); @@ -63,4 +64,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +// +//db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false}); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy new file mode 100644 index 000000000..9da03d468 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author shenjian + * @since 2021/5/17 + */ +class SavePoi { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + AtomicInteger count = new AtomicInteger() + for (List listT : data) { + final list = listT + executorService.execute(new Runnable() { + @Override + void run() { + if (list.size() >= 17) { + def city = list.get(1) + def lng = Double.valueOf(list.get(11)) + def lat = Double.valueOf(list.get(12)) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("status", 1) + obj.put("flag", "距离异常入库") + } + conn.insert(arounds) + println(count.incrementAndGet()) + } + } + } + }) + }; + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy index 3037ee807..f5c6f3504 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -75,16 +75,5 @@ class DistanceDeal { // } } - static def parserByGaode(def city, def address) { - JSONObject json = GaoDeApiUtils.geo(city, address) - if (json && json.status == "1") { - def ret = json.geocodes - if (ret) { - def str = ret.get(0).location - def level = ret.get(0).level - def arrs = str.split(",") - return ["lng": arrs[0], "lat": arrs[1]] - } - } - } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 1ccbf1713..3e745dfd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import net.sf.json.JSONObject import java.util.concurrent.ConcurrentHashMap @@ -84,4 +86,21 @@ class GaoDeUtils { } return result } + + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..2f93a62e2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } +} From 964b25baa262e457816e1bc7338eea444a640f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 10:42:29 +0800 Subject: [PATCH 042/212] =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 155 +++++++----------- .../temp/shenjian/cqrcb/DealUnMatched.groovy | 106 ++++++++++++ .../temp/shenjian/cqrcb/ImportSchool.groovy | 53 ++++++ .../shenjian/cqrcb/MoveArroundTemp.groovy | 81 +++++++-- .../script/temp/shenjian/cqrcb/POIRule.groovy | 8 +- .../script/temp/shenjian/cqrcb/SavePoi.groovy | 55 +++++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 3 +- .../temp/shenjian/school/DistanceDeal.groovy | 13 +- .../temp/shenjian/util/GaoDeUtils.groovy | 19 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 32 ++++ 10 files changed, 400 insertions(+), 125 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 564da168d..5147a5b04 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -28,6 +28,7 @@ class AroundExportByNewWay { static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -42,9 +43,6 @@ class AroundExportByNewWay { try { def id = district.get(0) def district_code = district.get(2) - if (district_code == 'B100203') { - println(district_code) - } def name = district.get(3) List result = [] if (district.get(20) != null && district.get(21) != null) { @@ -57,24 +55,28 @@ class AroundExportByNewWay { q.put('status', 0) q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); - Map map = new ConcurrentHashMap<>(); poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) if (distance <= 3000) { - dealSpecial(obj, map, id, distance, result) + if (distance == 0) { + distance = 50 + } + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = formatType(obj.typecode) + ret[3] = distance.toInteger() + result << ret } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - def type = map.get(key).get("type") - def ret = [] - ret[0] = id - ret[1] = key - ret[2] = type - ret[3] = (distance / count1).toInteger() - result << ret - } + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) @@ -86,100 +88,63 @@ class AroundExportByNewWay { } }) }; - latch.await() + latch.await(); def headArr = ['IDB', '名称','类型', '距离'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + Map map = new HashedMap() + Integer key = 1; def ret = [] for (List district : data) { def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { ret.addAll(resultMap.get(district_code + name)) + if (ret.size() > 600000) { + map.put(key++, ret) + ret = [] + } } } - println(ret.size()) + map.put(key++, ret) //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); - FileUtils.batchCreateFile(ret, "F:\\数据\\重庆农商行\\周边配套.dat", headArr, "utf-8") -// EasyExcel.write("F:\\数据\\重庆农商行\\周边配套.xlsx").head(head).sheet("成交信息").doWrite(ret); + for (Integer k : map.keySet()) { + FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + //EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + //printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def id, double distance, def result) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("栋") != -1) { - return false - } - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false - } - } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("(出入口)") != -1) { - return false - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - distance + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - child.put("type", obj.type) - map.put(key, child) - } else { - child.put("distance", distance) - child.put("type", obj.type) - child.put("count", 1) - map.put(key, child) - } - } else { - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = obj.type - ret[3] = distance.toInteger() - result << ret + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String type) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy new file mode 100644 index 000000000..48be458a5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/DealUnMatched.groovy @@ -0,0 +1,106 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import com.mongodb.QueryOperators + +/** + * 处理未匹配的小区 + * + * @author shenjian + * @since 2021/5/17 + */ +class DealUnMatched { + + static void main(String[] args) { + // getLatLngByName() + // getLatLngByNumber() + //findExceptionDistance() + + // B114862 + def lng = 107.736591 + def lat = 29.872502 + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 500) + DBObject q = new BasicDBObject() + q.put('city', '重庆') + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + DBUtils.seaweed.find(q).batchSize(100).each {obj -> + println(obj) + } + } + + static void findExceptionDistance() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + for (List district : data) { + if (district.size() > 12) { + def lng = java.lang.Double.valueOf(district.get(8)) + def lat = java.lang.Double.valueOf(district.get(9)) + def lng1 = java.lang.Double.valueOf(district.get(11)) + def lat1 = java.lang.Double.valueOf(district.get(12)) + double distance = GaoDeUtils.getDistance(lat, lng, lat1, lng1) + if (distance > 500) { + println("距离异常" + district) + district.add(distance) + district.add("距离异常") + } + } + } + data.add(0, new ArrayList()) + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "周边配套") + } + + static void getLatLngByNumber() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",1) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + + //def region = SeaweedUtils.getRegionByCode(district_code) + if (district.size() <= 5) { + if (number != null && number.size() > 5) { + def map = GaoDeUtils.parserByGaode(city, number) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过楼栋得出") + } + } else { + println(district_code + " " + name + " " + number) + } + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配1") + } + + static void getLatLngByName() { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", 0) + for (List districtT : data) { + final List district = districtT + def city = district.get(1) + def district_code = district.get(2) + def name = district.get(3) + def number = district.get(4) + //def region = SeaweedUtils.getRegionByCode(district_code) + + def map = GaoDeUtils.parserByGaode(city, name) + if (map != null) { + district.add(map.get("lng")) + district.add(map.get("lat")) + district.add("通过小区得出") + } + } + ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", data, "未匹配") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy new file mode 100644 index 000000000..b8f0452ee --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/ImportSchool.groovy @@ -0,0 +1,53 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +/** + * 导入中学小学信息 + * + * @author shenjian + * @since 2021/5/17 + */ +class ImportSchool { + + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\重庆学校_0902.xlsx", 1) + List result = new ArrayList(); + for (List school : data) { + try { + def city = school.get(0) + def region = school.get(1) + def name = school.get(2) + def address = school.get(4) + def type = school.get(6) + def lng = school.get(9) + def lat = school.get(10) + + DBObject dbObject = new BasicDBObject() + dbObject.put("province", "重庆") + dbObject.put("city", city) + dbObject.put("region", region) + dbObject.put("name", name) + dbObject.put("address", address) + dbObject.put("type", type) + dbObject.put("typecode", "141200") + dbObject.put("lng", lng) + dbObject.put("lat", lat) + dbObject.put("status", 0) + dbObject.put("flag", "excel导入") + result.add(dbObject) + } catch(Exception ex) { + println(school) + ex.printStackTrace() + } + } + + poiConn.insert(result) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy index e17e12840..c641a4fc8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/MoveArroundTemp.groovy @@ -1,13 +1,17 @@ package com.fangjia.script.temp.shenjian.cqrcb +import cn.emay.mina.util.ConcurrentHashSet import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CopyOnWriteArrayList import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger /** * POI迁移 @@ -22,18 +26,20 @@ class MoveArroundTemp { static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - List list = Collections.synchronizedList(new ArrayList()) - List ids = conn.distinct("id") - CountDownLatch latch = new CountDownLatch(ids.size()) - for (String idT : ids) { - final String id = idT + Map map = new ConcurrentHashMap() + AtomicInteger count = new AtomicInteger() + int size = conn.count(new BasicDBObject("status", 1)) + println(size) + CountDownLatch latch = new CountDownLatch(size) + conn.find(new BasicDBObject("status", 1)).batchSize(10000).each { objT -> + final def obj = objT executorService.submit(new Runnable() { @Override void run() { try { - DBObject dbObject = new BasicDBObject() - conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> - dbObject.put("_id", id) + if (map.get(obj.id) == null && conn43.count(new BasicDBObject("_id", obj.id)) <= 0) { + DBObject dbObject = new BasicDBObject() + dbObject.put("_id", obj.id) dbObject.put("name", obj.name) dbObject.put("type", obj.type) dbObject.put("typecode", obj.typecode) @@ -44,21 +50,66 @@ class MoveArroundTemp { dbObject.put("province", obj.pname) dbObject.put("city", obj.cityname) dbObject.put("region", obj.adname) + dbObject.put("status", 2) + dbObject.put("flag", "待清洗") + map.put(obj.id, dbObject) } - list.add(dbObject) } catch(Exception ex) { ex.printStackTrace() } finally { latch.countDown() - println("当前记录数" + list.size()) + println(count.incrementAndGet()) } } }) - }; - latch.await(); - conn43.insert(list); - list.clear(); - println("同步清理完成"); + } + latch.await() + List ret = new ArrayList() + for (String key : map.keySet()) { + ret << map.get(key) + } + conn43.insert(ret) executorService.shutdown() } + + static origin() { +// List list = Collections.synchronizedList(new ArrayList()) +// List ids = conn.distinct("id") +// CountDownLatch latch = new CountDownLatch(ids.size()) +// for (String idT : ids) { +// final String id = idT +// executorService.submit(new Runnable() { +// @Override +// void run() { +// try { +// DBObject dbObject = new BasicDBObject() +// conn.find(new BasicDBObject("id", id)).limit(1).each { obj -> +// dbObject.put("_id", id) +// dbObject.put("name", obj.name) +// dbObject.put("type", obj.type) +// dbObject.put("typecode", obj.typecode) +// dbObject.put("address", obj.address) +// dbObject.put("lng", Double.valueOf(obj.location.split(",")[0])) +// dbObject.put("lat", Double.valueOf(obj.location.split(",")[1])) +// dbObject.put("tel", obj.tel) +// dbObject.put("province", obj.pname) +// dbObject.put("city", obj.cityname) +// dbObject.put("region", obj.adname) +// } +// list.add(dbObject) +// } catch(Exception ex) { +// ex.printStackTrace() +// } finally { +// latch.countDown() +// println("当前记录数" + list.size()) +// } +// } +// }) +// }; +// latch.await(); +// conn43.insert(list); +// list.clear(); +// println("同步清理完成"); +// executorService.shutdown() + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index e6ab78d3d..88033d0b8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -44,10 +44,11 @@ class POIRule { } // 更新规则 -// db.getCollection('gaode_address_poi').update({'name': /栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); +// db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCD]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); @@ -63,4 +64,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /智能柜/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); +// +//db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false}); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy new file mode 100644 index 000000000..9da03d468 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * @author shenjian + * @since 2021/5/17 + */ +class SavePoi { + + static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) + //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 + static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" + static def radius = "3000" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx",0) + AtomicInteger count = new AtomicInteger() + for (List listT : data) { + final list = listT + executorService.execute(new Runnable() { + @Override + void run() { + if (list.size() >= 17) { + def city = list.get(1) + def lng = Double.valueOf(list.get(11)) + def lat = Double.valueOf(list.get(12)) + List arounds = GaoDeUtils.searchNearByPoi(city, types, lng, lat, radius, "") + if (CollectionUtils.isNotEmpty(arounds)) { + for (BasicDBObject obj : arounds) { + obj.put("status", 1) + obj.put("flag", "距离异常入库") + } + conn.insert(arounds) + println(count.incrementAndGet()) + } + } + } + }) + }; + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index e5e264415..66703a1f0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -95,7 +95,8 @@ class SeaweedExport { for (def a : obj.alias) { temp.add(a) } - ret[4] = temp.join(",") + String als = temp.join(",") + ret[4] = als.length() > 20 ? "" : als } ret[5] = obj.car_site_count if (com.fangjia.script.datastore.FieldValidatorUtils.householdCountValidator(obj.household_count) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy index 3037ee807..f5c6f3504 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/school/DistanceDeal.groovy @@ -75,16 +75,5 @@ class DistanceDeal { // } } - static def parserByGaode(def city, def address) { - JSONObject json = GaoDeApiUtils.geo(city, address) - if (json && json.status == "1") { - def ret = json.geocodes - if (ret) { - def str = ret.get(0).location - def level = ret.get(0).level - def arrs = str.split(",") - return ["lng": arrs[0], "lat": arrs[1]] - } - } - } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 1ccbf1713..3e745dfd3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,7 +1,9 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import net.sf.json.JSONObject import java.util.concurrent.ConcurrentHashMap @@ -84,4 +86,21 @@ class GaoDeUtils { } return result } + + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..2f93a62e2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,32 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } +} From 034f924f509720dd9c16ff8122a0619e3ecb5b6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 17 May 2021 18:39:41 +0800 Subject: [PATCH 043/212] =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86?= =?UTF-8?q?=E8=A1=8C=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 65 ++++++++++++------- .../script/temp/shenjian/cqrcb/POIRule.groovy | 2 +- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 5147a5b04..4ec4dac97 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -4,6 +4,7 @@ import com.alibaba.excel.EasyExcel import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.district.Distance import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -45,28 +46,23 @@ class AroundExportByNewWay { def district_code = district.get(2) def name = district.get(3) List result = [] - if (district.get(20) != null && district.get(21) != null) { + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) - // 周围半径为3千米 - def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) - DBObject q = new BasicDBObject() - q.put('status', 0) - q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); - q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); - poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> - double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) - if (distance <= 3000) { - if (distance == 0) { - distance = 50 - } - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = formatType(obj.typecode) - ret[3] = distance.toInteger() - result << ret + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id) } } } else { @@ -81,7 +77,7 @@ class AroundExportByNewWay { resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) } catch (Exception ex) { - + ex.printStackTrace() } finally { latch.countDown() } @@ -113,14 +109,37 @@ class AroundExportByNewWay { map.put(key++, ret) //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); for (Integer k : map.keySet()) { - FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") - //EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); } - //printUnMatched(unMatchedMap) + printUnMatched(unMatchedMap) executorService.shutdown() } + static calcAroundInfo(def lat, def lng, def result, def id) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 3000) { + if (distance == 0) { + distance = 50 + } + def ret = [] + ret[0] = id + ret[1] = obj.name + ret[2] = formatType(obj.typecode) + ret[3] = distance.toInteger() + result << ret + } + } + } + static printUnMatched(def unMatchedMap) { def ret = [] ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index 88033d0b8..5deadb989 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -48,7 +48,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[ABCD]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'name': /[ABCDEFHG]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); From 113f2e8c506154773725fc21e51f431b0d19ad5e Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Mon, 17 May 2021 20:58:57 +0800 Subject: [PATCH 044/212] =?UTF-8?q?=E5=B0=8F=E5=8C=BAmaster=E5=88=86?= =?UTF-8?q?=E6=94=AF=E5=BC=80=E5=8F=91=E5=90=88=E5=B9=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/company/AddCompany.groovy | 71 +++++ .../shenjian/company/DevelopCompany.groovy | 33 ++ .../shenjian/company/PropertyCompany.groovy | 34 +++ .../temp/shenjian/dto/SearchAddressDTO.groovy | 16 + .../fill/CalcCubageAndVirescence.groovy | 116 +++++++ .../shenjian/seaweed/BuildStructure.groovy | 112 +++++++ .../temp/shenjian/seaweed/BuildType.groovy | 129 ++++++++ .../temp/shenjian/seaweed/BuildYear.groovy | 91 ++++++ .../temp/shenjian/seaweed/CollectTest.groovy | 87 ++++++ .../temp/shenjian/seaweed/Company.groovy | 155 ++++++++++ .../script/temp/shenjian/seaweed/Fee.groovy | 112 +++++++ .../temp/shenjian/seaweed/HousePurpose.groovy | 109 +++++++ .../temp/shenjian/seaweed/HouseType.groovy | 99 ++++++ .../temp/shenjian/seaweed/Position.groovy | 78 +++++ .../temp/shenjian/seaweed/Seaweed.groovy | 288 ++++++++++++++++++ .../script/temp/shenjian/seaweed/Temp.groovy | 28 ++ .../temp/shenjian/util/DistrictUtils.groovy | 55 ++++ 17 files changed, 1613 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy new file mode 100644 index 000000000..1c8d2f70f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/AddCompany.groovy @@ -0,0 +1,71 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils; + +/** + * + * 新增公司 + * + * @author shenjian + * @since 2021-03-16 + */ +class AddCompany { + + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static final Set developCompanySet = new HashSet() + static final Set propertyCompanySet = new HashSet() + + static final Set actualDevelopCompanySet = new HashSet() + static final Set actualPropertyCompanySet = new HashSet() + + static { + develop_company.find().batchSize(10000).each { obj -> + developCompanySet.add(obj.company) + } + property_company.find().batchSize(10000).each { obj -> + propertyCompanySet.add(obj.company) + } + } + + static void main(String[] args) { + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "哈尔滨") + DbUtils.seaweed.find(query, + new BasicDBObject("developer", 1).append("estate_company", 1)) + .batchSize(10000).each {obj -> + actualDevelopCompanySet.add(obj.developer) + actualPropertyCompanySet.add(obj.estate_company) + } + + def developCompanySaves = [] + def propertyCompanySaves = [] + for (String name : actualDevelopCompanySet) { + if (StringUtils.isBlank(name) || developCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + developCompanySaves << item + } + for (String name : actualPropertyCompanySet) { + if (StringUtils.isBlank(name) || propertyCompanySet.contains(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + propertyCompanySaves << item + } + + + develop_company.insert(developCompanySaves) + property_company.insert(propertyCompanySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy new file mode 100644 index 000000000..fb1e94c2b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/DevelopCompany.groovy @@ -0,0 +1,33 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils; + +/** + * + * 开发商公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class DevelopCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection develop_company = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static void main(String[] args) { + def companySaves = [] + List result = seaweed.distinct("developer") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + develop_company.insert(companySaves) + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy new file mode 100644 index 000000000..dca212b14 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/company/PropertyCompany.groovy @@ -0,0 +1,34 @@ +package com.fangjia.script.temp.shenjian.company + +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils + +/** + * 物业公司 + * + * @author shenjian + * @since 2021-03-13 + */ +class PropertyCompany { + + static final DBCollection seaweed = MongoUtils.getCollectionByName("fangjia", "seaweed", "192.168.0.136") + static final DBCollection property_company = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + + static void main(String[] args) { + // 物业公司 + def companySaves = [] + List result = seaweed.distinct("estate_company") + for (String name : result) { + if (StringUtils.isBlank(name)) { + continue; + } + def item = new BasicDBObject() + item.company = name + companySaves << item + } + property_company.insert(companySaves) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy new file mode 100644 index 000000000..01b1c1f27 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/SearchAddressDTO.groovy @@ -0,0 +1,16 @@ +package com.fangjia.script.temp.shenjian.dto + +/** + * 匹配返回结构体 + */ +class SearchAddressDTO { + + String code; + String msg; + Result result; + + class Result { + String street + String streetNumber + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy new file mode 100644 index 000000000..ada6e667b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/fill/CalcCubageAndVirescence.groovy @@ -0,0 +1,116 @@ +package com.fangjia.script.temp.shenjian.fill + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.GeoHelper +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 根据周围500米容积率与绿化率平均值推算小区 + * + * @author shenjian* + * @since 2021-03-13 + */ +@Slf4j +class CalcCubageAndVirescence { + + static main(args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("CalcCubageAndVirescence") + log.info("开始根据周围500米容积率与绿化率平均值推算小区") + def cities = ['北京', '重庆', '哈尔滨'] + for (def city : cities) { + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts); + def result = [] + districts.each { obj -> + if (obj.finish_building_date) { + int year = CommonUtils.getBuildYear(obj.finish_building_date) + if (year > 2000) { + if (!(obj.cubage_percent && obj.virescence_percent)) { + def temp = this.findResult(obj.lat2 / 10000000000.0d, obj.lng2 / 10000000000.0d, 500, sameRegionDistricts.get(obj.region), obj.name, obj._id) + result << temp + } + + } + } + } + def headers = ["小区ID", "容积率", "绿化率"] + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\补充容积绿化" + city + ".csv", headers, "UTF-8") + } + stopWatch.stop() + log.info("完成根据周围500米容积率与绿化率平均值推算, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + } + + static def findResult(lat, lng, radius, districts, region, id) { + def avgMap = [:] + def ret = [id] + if (lat && lng) { + def cubagePercentArr = [] + def virescencePercentArr = [] + def map = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, radius) + for (def row : districts) { + if (row.lat2 && row.lng2) { + def lat2 = row.lat2 / 10000000000.0; + def lng2 = row.lng2 / 10000000000.0; + if (lat2 >= map.minLat && lat2 <= map.maxLat && lng2 >= map.minLng && lng2 <= map.maxLng) { + def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2) + if (distance <= radius) { + if (row.cubage_percent) { + cubagePercentArr << row.cubage_percent; + } + if (row.virescence_percent) { + virescencePercentArr << row.virescence_percent + } + } + } + } + } + try { + // 计算平均容积率 + if (cubagePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : cubagePercentArr) { + sum = sum + Double.valueOf(m) + count++ + } + if (sum) { + double avgCubagePercent = sum / count + ret << DataHandleUtils.getDoubleRound(2, avgCubagePercent) + } + } else { + ret << "" + } + + // 计算平均绿化率 + if (virescencePercentArr) { + double sum = 0 + double count = 0.0d + for (def m : virescencePercentArr) { + sum += Double.valueOf(m) + count++ + } + if (sum) { + def avgVirescencePercent = sum / (count) + ret << DataHandleUtils.getDoubleRound(2, avgVirescencePercent) + } + } else { + ret << "" + } + log.info("找到可以优化的小区" + region + "周围平均值为:容积率" + ret[1] + ", 绿化率" + ret[2]) + } catch(Exception ex) { + log.error("容积率或绿化率不标准,无法解析") + } + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy new file mode 100644 index 000000000..278446f50 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildStructure.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑结构 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildStructure { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildStructureMap = new HashMap() + static { + buildStructureMap.put("钢结构", "3") + buildStructureMap.put("钢、钢筋混凝土结构", "1") + buildStructureMap.put("钢筋混凝土结构", "1") + buildStructureMap.put("钢混", "1") + buildStructureMap.put("钢混结构", "1") + buildStructureMap.put("混合结构", "11") + buildStructureMap.put("钢筋混凝土", "11") + buildStructureMap.put("砖木结构", "10") + buildStructureMap.put("其他结构", "11") + buildStructureMap.put("砖混结构", "2") + buildStructureMap.put("框架结构", "4") + buildStructureMap.put("框架剪力墙结构", "5") + buildStructureMap.put("剪力墙结构", "6") + buildStructureMap.put("筒体结构", "7") + buildStructureMap.put("框架筒体结构", "8") + buildStructureMap.put("筒中筒", "9") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_structure", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑结构","备注", "创建时间", "更新时间", "来源公司"] + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑结构\\bigdata_commbuildstru_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑结构表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getBuildStructure(dbo.get("build_structure")) // 建筑结构 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("build_structure"))) { + rows << row + count.incrementAndGet() + } + }) + return rows + }; + + static def getBuildStructure(def houseStructure) { + def ret + if (StringUtils.isNotBlank(houseStructure)) { + ret = buildStructureMap.get(houseStructure) + if (StringUtils.isBlank(ret)) { + ret = "11" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy new file mode 100644 index 000000000..408b87f48 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildType.groovy @@ -0,0 +1,129 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map buildTypeMap = new HashMap() + static { + buildTypeMap.put("多层", "0104") + buildTypeMap.put("小高层", "0103") + buildTypeMap.put("高层", "0199") + buildTypeMap.put("超高层", "0199") + buildTypeMap.put("花园洋房", "0199") + buildTypeMap.put("老洋房", "0199") + buildTypeMap.put("四合院", "0199") + buildTypeMap.put("独栋", "0105") + buildTypeMap.put("双拼", "0199") + buildTypeMap.put("联排", "0106") + buildTypeMap.put("叠加", "0107") + buildTypeMap.put("老式别墅洋房", "0199") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区建筑类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑类型","备注", "创建时间", "更新时间", "来源公司"]; + def seaweedCount = [21596, 17053, 5410]; + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑类型\\bigdata_commbuildtype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + if (StringUtils.isNotBlank(dbo.get("build_type"))) { + Set set = this.getBuildTypeCode(dbo.get("build_type")) + for (String type : set) { + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << type // 建筑类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + } + if (set.size() > 0) { + count.incrementAndGet() + } + + } + }) + return rows + } + + + static Set getBuildTypeCode(def buildType) { + Set set = new HashSet<>() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + set.add(buildTypeMap.get(type)) + } + if (set.isEmpty()) { + set.add("0199") // 其他 + } + } + return set + } + + static List getBuildTypeCodeSeq(def buildType) { + List list = new ArrayList() + if (StringUtils.isNotBlank(buildType)) { + String[] types = buildType.split(",") + for (String type : types) { + list.add(buildTypeMap.get(type)) + } + if (list.isEmpty()) { + list.add("0199") // 其他 + } + } + return list + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy new file mode 100644 index 000000000..cee15c27c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/BuildYear.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.text.SimpleDateFormat +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区建筑年代表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class BuildYear { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildYear") + log.info("开始导出小区建筑年代表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("finish_building_date", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","建筑年代","备注", "创建时间", "更新时间", "来源公司"] + + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\建筑年代\\bigdata_commbuildyear_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区建筑年代表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + Integer year = CommonUtils.getBuildYear(dbo.get("finish_building_date")) // 建筑年代 + if (year > 1900) { + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << year + row << "" // 备注 + + Integer createYear = CommonUtils.getBuildYear(DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy")) + if (year > createYear) { + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()); + row << dbo.get("finish_building_date") + " " + new SimpleDateFormat("hh:MM:ss").format(new Date()) + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } + row << "fjw" // 来源公司 + + rows << row + count.incrementAndGet() + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy new file mode 100644 index 000000000..e5cbce7af --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CollectTest.groovy @@ -0,0 +1,87 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.ExcelUtil +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils + +/** + * @author shenjian + * @since 2021-03-13 + */ +class CollectTest { + + static void main(String[] args) { + def cites = ["北京", "重庆", "哈尔滨"]; + for (def city : cites) { + /** + * _id 小区ID + * name 小区名 + * alias 小区曾用名 + */ + def districts = this.findAllCityDistricts(city, "district", + ["city", "_id", "region", "name", "alias"]) + print(districts) + } + } + + static def findAllCityDistricts(def city, def cat, def fields) { + DBObject q = new BasicDBObject(); + q.put("visible", 0) + q.put("cat", cat) + q.put("region", "密云") + if (city) { + q.put("city", city) + } + + DBObject keys = new BasicDBObject(); + if (fields) { + for (def field : fields) { + keys.put(field, 1) + } + } + + DSCursor cursor = CommonDatautils.getDataStoreConnection().find(q, keys).batchSize(4000) + def districtKeys = [] + def rows = []; + rows << ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + + for (def dbo : cursor) { + def row = []; + row << null // TODO + row << dbo.get("_id") + row << dbo.get("name") + row << dbo.get("alias") + row << null // TODO + row << dbo.get("street_community") // 街道 TODO 拆分为街道门牌号 + row << dbo.get("street_community") // 门牌号 + rows << dbo.get("address") + rows << dbo.get("all_area") + rows << dbo.get("build_area") + + rows << dbo.get("land_property") + ExcelUtils.write("d:\\数据\\小区表.xlsx", rows, "小区") + + StringBuilder data = new StringBuilder() + Set keySet = dbo.keySet() + keySet.forEach({key -> + if ("alias" == key) { + def value = dbo.get(key) + if (value != null) { + value = value[0] + } + data.append(value) + } else { + data.append(dbo.get(key)) + } + data.append("|@|") + }) + districtKeys << StringUtils.substringBeforeLast(data.toString(), "|@|") + }; + return districtKeys; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy new file mode 100644 index 000000000..87b609daa --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Company.groovy @@ -0,0 +1,155 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang3.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区公司表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Company { + + private static ExecutorService executorService = Executors.newFixedThreadPool(2); + + // 物业公司 + static final DBCollection PROPERTY_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "property_company", "192.168.0.136") + // 开发商 + static final DBCollection DEVELOP_COMPANY_COLLECTION = MongoUtils.getCollectionByName("fangjia", "develop_company", "192.168.0.136") + + static final propertyCompany = new HashMap() + static final developCompany = new HashMap() + + static { + PROPERTY_COMPANY_COLLECTION.find().forEach({company -> + propertyCompany.put(company.get("company"), company.get("_id").toString()) + }) + + DEVELOP_COMPANY_COLLECTION.find().forEach({company -> + developCompany.put(company.get("company"), company.get("_id").toString()) + }) + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Company") + log.info("开始导出小区公司表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("developer", 1) + .append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","公司ID","公司类型", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger developCount = new AtomicInteger(0) + AtomicInteger estateCount = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, developCount, estateCount) + println("城市" + city + " " + cityId + "物业数量为" + estateCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((estateCount.get() * 100) / seaweedCount[i])) ) + println("城市" + city + " " + cityId + "开发商数量为" + developCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((developCount.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\公司\\bigdata_commcomp_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区公司表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger developCount, AtomicInteger estateCount) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + def rowCanSave = true + def row1CanSave = true + row << cityId // 城市ID + row << dbo.get("_id").toString() + def propertyCompany = dbo.get("estate_company") + if (StringUtils.isNotBlank(propertyCompany)) { + row << this.getPropertyCompanyId(propertyCompany) // 公司ID + row << "SV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + rowCanSave = false + } + + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + def row1 = [] + row1 << cityId + row1 << dbo.get("_id").toString() + def developerCompany = dbo.get("developer") + if (StringUtils.isNotBlank(developerCompany)) { + row1 << this.getDevelopCompanyId(developerCompany) // 公司ID + row1 << "DV" // 公司类型(物业管理公司SV、 房地产开发商DV ) + } else { + row1CanSave = false + } + + row1 << "" // 备注 + row1 << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row1 << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row1 << "fjw" // 来源公司 + if (rowCanSave) { + rows << row + estateCount.incrementAndGet() + } + if (row1CanSave) { + rows << row1 + developCount.incrementAndGet() + } + }) + return rows + } + + /** + * 获取开发商公司ID + * @param company 公司名称 + * @return + */ + static def getDevelopCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return developCompany.get(company) + } + + /** + * 获取物业公司ID + * @param company 公司名称 + * @return + */ + static def getPropertyCompanyId(String company) { + if (StringUtils.isBlank(company)) { + return "" + } + return propertyCompany.get(company) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy new file mode 100644 index 000000000..614da8edb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Fee.groovy @@ -0,0 +1,112 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger + +/** + * 小区费用表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Fee { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Fee") + log.info("开始导出小区费用表") + def cites = ["北京", "重庆", "哈尔滨"] + def seaweedCount = [21596, 17053, 5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_company", 1).append("estate_charge", 1) + .append("estate_type2", 1).append("build_type", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID", "公司ID", "费用", "房屋用途","建筑类型", "小区费用类型", "费用说明", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + def city = cites[i] + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + AtomicInteger count = new AtomicInteger(0) + def result = this.findAllCityDistricts(city, "district", cityId, filter, count) + println("城市" + city + " " + cityId + "数量为" + count.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((count.get() * 100) / seaweedCount[i])) ) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\费用\\bigdata_commfee_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区费用表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, AtomicInteger count) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def estate_charge = dbo.get("estate_charge") + if (estate_charge && estate_charge.toString().indexOf("~") != -1) { + String[] arr = estate_charge.toString().split("~") + estate_charge = ((Double.valueOf(arr[0]) + Double.valueOf(arr[1])) / 2.0D).toString() + } + if (estate_charge == '-' || estate_charge == '无') { + estate_charge = "" + } + if (StringUtils.isNotBlank(dbo.get("estate_company")) && StringUtils.isNotBlank(estate_charge) + && StringUtils.isNotBlank(dbo.get("estate_type2")) && StringUtils.isNotBlank(dbo.get("build_type"))) { + List buildTypes = BuildType.getBuildTypeCodeSeq(dbo.get("build_type")) + Set savedBuildTypeSet = new HashSet<>() + boolean isFirst = true + for (String buildType : buildTypes) { + if (savedBuildTypeSet.contains(buildType)) { + continue + } + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() // 小区ID + row << Company.getPropertyCompanyId(dbo.get("estate_company")) // 公司ID + + // 处理 db.getCollection('seaweed').find({'estate_charge': '3.3~4.3'}, {'estate_charge': 1}) + // 特殊费用 3.3~4.3 + if (isFirst) { + row << estate_charge // 费用 + isFirst = false + count.incrementAndGet() + } else { + row << "" + break + } + row << HousePurpose.getHousePurposeCode(dbo.get("estate_type2")) // 房屋用途 + row << buildType // 建筑类型 + row << "1" // 小区费用类型 物业费1 电梯费2 垃圾费3 + row << dbo.get("") // 费用说明 + row << dbo.get("") // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + rows << row + + savedBuildTypeSet.add(buildType) + } + } + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy new file mode 100644 index 000000000..ba8dc7c2e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HousePurpose.groovy @@ -0,0 +1,109 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.StringUtil +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋用途表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HousePurpose { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map housePurposeMap = new HashMap() + static { + housePurposeMap.put("普通住宅", "11") + housePurposeMap.put("别墅", "11") + housePurposeMap.put("商住", "13") + housePurposeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("BuildStructure") + log.info("开始导出小区房屋用途表表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("household_count", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","小区ID","房屋用途","户数", "备注", "创建时间", "更新时间", "来源公司"]; + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋用途\\bigdata_commhouseuse_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋用途表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << getHousePurposeCode(dbo.get("estate_type2")) + + if (dbo.get("household_count") != null) { + if (Long.valueOf(dbo.get("household_count")) < 50) { + row << "" // 户数 + } else { + row << dbo.get("household_count") + } + } else { + row << "" // 户数 + } + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHousePurposeCode(def housePurpose) { + def ret + if (StringUtils.isNotBlank(housePurpose)) { + ret = housePurposeMap.get(housePurpose) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy new file mode 100644 index 000000000..8e77760a9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/HouseType.groovy @@ -0,0 +1,99 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区房屋类型表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class HouseType { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static Map houseTypeMap = new HashMap() + static { + houseTypeMap.put("普通住宅", "01") + houseTypeMap.put("商住", "07") + houseTypeMap.put("别墅", "08") + houseTypeMap.put("其他", "99") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("HouseType") + log.info("开始导出小区房屋类型表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("estate_type2", 1).append("c_date", 1).append("m_date", 1) + + def headers = ["城市ID","小区ID","小区房屋类型","备注", "创建时间", "更新时间", "来源公司"] + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\房屋类型\\bigdata_commhousetype_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区房屋类型表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << this.getHouseTypeCode(dbo.get("estate_type2")) // 小区房屋类型 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + if (dbo.get("m_date")) { + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + } else { + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") + } + + row << "fjw" // 来源公司 + + if (StringUtils.isNotBlank(dbo.get("estate_type2"))) { + rows << row + } + }) + return rows + }; + + static def getHouseTypeCode(def houseType) { + def ret + if (StringUtils.isNotBlank(houseType)) { + ret = houseTypeMap.get(houseType) // 房屋类型 + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy new file mode 100644 index 000000000..269f01ab3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Position.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区地理位置表 + * + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Position { + + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Location") + log.info("开始导出小区地理位置表结构表") + def cites = ["北京", "重庆", "哈尔滨"] + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("lng", 1).append("lat", 1) + .append("c_date", 1).append("m_date", 1) + def headers = ["城市ID","物体类型","物体ID","位置类型", "序号", "经度", "纬度", "海拔", "备注", "创建时间", "更新时间", "来源公司"] + + for (def city : cites) { + def cityId = FangJiaAreaMapUtils.convertToOfficial(city).get("city_code") // 城市ID + def result = this.findAllCityDistricts(city, "district", cityId, filter) + executorService.execute(new Runnable() { + @Override + void run() { + FileUtils.batchCreateFile(result, "d:\\数据\\地理位置\\bigdata_position_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区地理位置表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter) { + def rows = [] + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + def row = [] + row << cityId // 城市ID + row << "5" // 物体类型 小区5 + row << dbo.get("_id").toString() // 物体ID + row << "1" // 位置类型 中心点1 + row << "1" // 序号 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lng") / 10000000000d) // 经度 + row << DataHandleUtils.getDoubleRound(6, dbo.get("lat") / 10000000000d) // 纬度 + row << "" // 海拔 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + rows << row + }) + return rows + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy new file mode 100644 index 000000000..0c2f16c13 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Seaweed.groovy @@ -0,0 +1,288 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.temp.shenjian.dto.SearchAddressDTO +import com.fangjia.script.temp.shenjian.fill.CalcCubageAndVirescence +import com.fangjia.script.temp.shenjian.util.DistrictUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.FangJiaAreaMapUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import groovy.util.logging.Slf4j +import org.apache.commons.lang.StringUtils +import org.springframework.util.StopWatch + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.regex.Pattern + +/** + * @author shenjian + * @since 2021-03-13 + */ +@Slf4j +class Seaweed { + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + private static ExecutorService executorService = Executors.newFixedThreadPool(3); + private static HttpRequest http = new HttpRequest() + private static final String TOKEN = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + private static Map seaweedTypeMap = new HashMap(); + private static Pattern pattern = Pattern.compile("\\s*|\t|\r|\n"); // 取消空格换行等 + private static Map cityDistrictMap = new HashMap(); + + static { + seaweedTypeMap.put("普通住宅", "pa101") + seaweedTypeMap.put("别墅", "pa101") + seaweedTypeMap.put("商住", "pa101") + } + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("Seaweed") + log.info("开始导出小区公司表") + //def cites = ["北京", "重庆", "哈尔滨"] + //def seaweedCount = [21596, 17053, 5410]; + def cites = ["哈尔滨"] + def seaweedCount = [5410]; + DBObject filter = new BasicDBObject() + filter.append("_id", 1).append("name", 1).append("alias", 1).append("street_community", 1) + .append("address", 1).append("all_area", 1).append("build_area", 1).append("region", 1) + .append("estate_type2", 1).append("virescence_percent", 1).append("cubage_percent", 1) + .append("car_site_count", 1).append("c_date", 1).append("m_date", 1) + .append("lng", 1).append("lat", 1) + def headers = ["城市ID","小区ID","小区名称","小区曾用名","行政区ID","街道","门牌号","详细地址","占地总面积","建筑总面积", + "小区类型","是否新盘", "绿化率", "容积率", "车位数量", "小区介绍", "备注", "创建时间", "更新时间", "来源公司"] + for (int i = 0; i < cites.size(); i++) { + final t = i; + def city = cites[t] + def sameRegionDistricts = [:] + def districts = DistrictUtils.getAllDistricts("district", city, sameRegionDistricts) + BasicDBObject cityDistrictInfo = FangJiaAreaMapUtils.convertToOfficial(city) + def cityId = cityDistrictInfo.get("city_code") // 城市ID + final def map = this.findAllCityDistricts(city, "district", cityId, filter, sameRegionDistricts) + executorService.execute(new Runnable() { + @Override + void run() { + def result = [] + AtomicInteger virescenceCount = new AtomicInteger(0) // 绿化率 + AtomicInteger cubageCount = new AtomicInteger(0) // 容积率 + AtomicInteger allAreaCount = new AtomicInteger(0) + AtomicInteger buildAreaCount = new AtomicInteger(0) + for (def key : map.keys()) { + result << map.get(key) + if (map.get(key) && map.get(key)[8] != "") { + allAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[9] != "") { + buildAreaCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[12] != "") { + virescenceCount.incrementAndGet() + } + if (map.get(key) && map.get(key)[13] != "") { + cubageCount.incrementAndGet() + } + } + println("城市" + city + " " + cityId + "绿化率数量为" + virescenceCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((virescenceCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "容积率数量为" + cubageCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((cubageCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "占地面积数量为" + allAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((allAreaCount.get() * 100) / seaweedCount[t])) ) + println("城市" + city + " " + cityId + "建筑面积数量为" + buildAreaCount.get() + ",覆盖率为" + DataHandleUtils.getDoubleRound(2, ((buildAreaCount.get() * 100) / seaweedCount[t])) ) + FileUtils.batchCreateFile(result, "d:\\数据\\小区\\bigdata_community_" + cityId + ".dat", headers, "UTF-8") + } + }) + } + stopWatch.stop() + log.info("完成导出小区表, 共耗时 {} 分", stopWatch.getTotalTimeSeconds() / 60) + executorService.shutdown() + fixedThreadPool.shutdown() + } + + static def findAllCityDistricts(def city, def cat, def cityId, def filter, def sameRegionDistricts) { + def map = new ConcurrentHashMap(); + DBObject query = new BasicDBObject() + query.append("cat", cat).append("visible", 0).append("city", city) + def total = DbUtils.seaweed.find(query, filter).count() + final CountDownLatch latch = new CountDownLatch(total) + DbUtils.seaweed.find(query, filter).batchSize(10000).each({dbo -> + fixedThreadPool.execute(new Runnable() { + @Override + void run() { + try { + dealSpecialChar(dbo) + def row = [] + + row << cityId // 城市ID + row << dbo.get("_id").toString() + row << dbo.get("name") + if (dbo.get("alias") != null && dbo.get("alias")[0] != null) { + row << pattern.matcher(dbo.get("alias")[0]).replaceAll("") + } else { + row << "" + } + def districtId + if (StringUtils.isBlank(cityDistrictMap.get(dbo.get("region")))) { + districtId = FangJiaAreaMapUtils.convertToOfficial(city, dbo.get("region")).get("district_code") + // 行政区ID + cityDistrictMap.put(dbo.get("region"), districtId) + } else { + districtId = cityDistrictMap.get(dbo.get("region")) + } + row << districtId + Map params = new HashMap(); + params.put("token", TOKEN); + params.put("city", city); + params.put("address", dbo.get("address")); + // 暂时不用解析 +// String s = http.get("http://open.fangjia.com/address/standardize", params, "utf-8"); +// SearchAddressDTO searchAddress = com.alibaba.fastjson.JSONObject.parseObject(s, SearchAddressDTO.class) +// if (searchAddress.code == '200') { +// //row << searchAddress.result.street // 街道 +// row << "" +// row << searchAddress.result.streetNumber // 门牌号 +// } else { +// row << "" +// row << "" +// } + row << "" // 街道 + row << "" // 门牌号 + + row << dbo.get("address") + + if (dbo.get("all_area") && dbo.get("all_area").toString().indexOf("公顷") != -1) { + def val = dbo.get("all_area").toString() + def temp = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000).longValue() + if (temp > 100000) { + row << "" + } else { + row << temp + } + + } else { + if (dbo.get("all_area")) { + if (Double.valueOf(dbo.get("all_area")) > 100000) { + row << "" + } else { + row << dbo.get("all_area") + } + } else { + row << "" + } + } + if (dbo.get("build_area") && Double.valueOf(dbo.get("build_area")) > 1000) { + row << dbo.get("build_area") + } else { + row << "" + } + + row << "pa101" // 小区类型 + row << "" // 是否新盘 + + if (dbo.get("virescence_percent") != null && dbo.get("cubage_percent") != null) { + // 绿化率 + if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) < 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 100 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) == 1) { + row << Double.valueOf(dbo.get("virescence_percent")) * 10 + } else if (dbo.get("virescence_percent") && Double.valueOf(dbo.get("virescence_percent")) >= 70) { + row << 65 + } else { + row << Double.valueOf(dbo.get("virescence_percent")) + } + + if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 7 && Double.valueOf(dbo.get("cubage_percent")) < 8)) { + row << 6.2 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 8 && Double.valueOf(dbo.get("cubage_percent")) < 9)) { + row << 6.3 + } else if (dbo.get("cubage_percent") && (Double.valueOf(dbo.get("cubage_percent")) >= 9 && Double.valueOf(dbo.get("cubage_percent")) < 10)) { + row << 6.5 + } else if (dbo.get("cubage_percent") && Double.valueOf(dbo.get("cubage_percent")) >= 10) { + row << 6.7 + } else { + row << dbo.get("cubage_percent") // 容积率 + } + } else { + def temp = CalcCubageAndVirescence.findResult(dbo.lat / 10000000000.0d, dbo.lng / 10000000000.0d, 500, sameRegionDistricts.get(dbo.region), dbo.name, dbo._id) + if (temp[2] && Double.valueOf(temp[2]) < 1) { + row << Double.valueOf(temp[2]) * 100 + } else if (temp[2] && Double.valueOf(temp[2]) == 1) { + row << Double.valueOf(temp[2]) * 10 + } else if (temp[2] && Double.valueOf(temp[2]) >= 70) { + row << 65 + } else { + row << temp[2] // 绿化率 + } + + if (temp[1] && (Double.valueOf(temp[1]) >= 7 && Double.valueOf(temp[1]) < 8)) { + row << 6.2 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 8 && Double.valueOf(temp[1]) < 9)) { + row << 6.3 + } else if (temp[1] && (Double.valueOf(temp[1]) >= 9 && Double.valueOf(temp[1]) < 10)) { + row << 6.5 + } else if (temp[1] && Double.valueOf(temp[1]) >= 10) { + row << 6.7 + } else { + row << temp[1] // 容积率 + } + } + if (dbo.get("car_site_count") != null) { + if (Long.valueOf(dbo.get("car_site_count")) > 2000 || Long.valueOf(dbo.get("car_site_count")) < 100) { + row << "" + } else { + row << dbo.get("car_site_count") // 车位数量 + } + } else { + row << "" // 车位数量 + } + + row << "" // 介绍 + row << "" // 备注 + row << DateUtils.formatDateByPattern(dbo.get("c_date"), "yyyy-MM-dd hh:MM:ss") // 创建时间 + row << DateUtils.formatDateByPattern(dbo.get("m_date"), "yyyy-MM-dd hh:MM:ss") // 更新时间 + row << "fjw" // 来源公司 + + map.put(dbo.get("_id").toString(), row) + } catch(Exception e) { + + } finally { + latch.countDown() + } + } + }) + }) + latch.await() + return map + }; + + /** + * 处理特殊字符 + * @param dbo + */ + static def dealSpecialChar(DBObject dbo) { + Set keySet = dbo.keySet() + for (String key : keySet) { + if (String.isInstance(dbo.get(key))) { + dbo.put(key, pattern.matcher(dbo.get(key)).replaceAll("")) + } + } + }; + + static def getSeaweedTypeCode(def value) { + def ret + if (StringUtils.isNotBlank(value)) { + ret = seaweedTypeMap.get(value) + if (StringUtils.isBlank(ret)) { + ret = "99" // 其他 + } + } else { + ret = "" + } + return ret + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy new file mode 100644 index 000000000..d85ddf81f --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy @@ -0,0 +1,28 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +class Temp { + + static main (args){ + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0).append("city", "北京") + + DBObject filter = new BasicDBObject() + filter.append("build_structure", 1) + + Set set = new HashSet<>() + + DbUtils.seaweed.find(query, filter).batchSize(10000).each({ dbo -> + if (dbo.build_structure) { + set.add(dbo.build_structure) + } + }) + + set.each {val -> + println(val) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy new file mode 100644 index 000000000..03d95a32c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/DistrictUtils.groovy @@ -0,0 +1,55 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.datastore.driver.DSCursor +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * 区域工具类 + */ +class DistrictUtils { + + static main(args) { + } + + static def getAllDistricts(def cat, def city, def sameRegionDistricts){ + DBObject query = new BasicDBObject(); + query.append("cat", cat).append("visible", 0).append("city", city) + + DBObject keys = new BasicDBObject().append("city", 1).append("region", 1).append("cubage_percent", 1) + .append("virescence_percent", 1).append("lat2", 1).append("lng2", 1) + .append("finish_building_date", 1).append("name", 1).append("_id", 1); + + DSCursor cursor = DbUtils.seaweed.find(query, keys).batchSize(10000) + Iterator iters = cursor.iterator() + def result = [] + while(iters.hasNext()) { + DBObject obj = iters.next() + + def map = [:]; + map.city = obj.city + map.cubage_percent = obj.cubage_percent; + map.virescence_percent = obj.virescence_percent; + map.lat2 = obj.lat2 + map.lng2 = obj.lng2 + map.region = obj.region + map.finish_building_date = obj.finish_building_date + map.name = obj.name + map._id = obj._id + result << map; + if (sameRegionDistricts.get(obj.region)) { + def temp = sameRegionDistricts.get(obj.region) + temp << map + sameRegionDistricts.put(obj.region, temp) + } else { + def temp = [] + temp << map + sameRegionDistricts.put(obj.region, temp) + } + } + return result; + } +} From b4c7f8ff1a9824291bdd3d20aa2c019f323d233f Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Mon, 17 May 2021 21:15:45 +0800 Subject: [PATCH 045/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E7=BB=BF=E5=8C=96?= =?UTF-8?q?=E7=8E=87=E3=80=81=E5=AE=B9=E7=A7=AF=E7=8E=87=E6=A0=BC=E5=BC=8F?= =?UTF-8?q?=E8=AF=9D=E5=B7=A5=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/util/SeaweedUtils.groovy | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 2f93a62e2..22969a8c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.temp.shenjian.util import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.apache.commons.lang.math.NumberUtils import java.util.concurrent.ConcurrentHashMap @@ -29,4 +30,73 @@ class SeaweedUtils { } return region } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && NumberUtils.isDigits(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (virescence_percent == 1) { + return virescence_percent * 10 + } + if (virescence_percent >= 70) { + return 65 + } + return virescence_percent + } + return "" + } + + /** + * 标准化绿化率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && NumberUtils.isDigits(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (cubage_percent && val >= 10) { + return 6.7 + } + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && NumberUtils.isDigits(car_site_count)) { + long val = Long.valueOf("car_site_count") + if (val > 2000 || val < 100) { + return "" + } + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && NumberUtils.isDigits(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } } From 6ec617af10d0b60ca2c5f6756bbcffbbd90e6d81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 18 May 2021 10:03:59 +0800 Subject: [PATCH 046/212] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E5=B7=A5=E5=85=B7?= =?UTF-8?q?=E7=B1=BB=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 14 ++++ .../script/temp/shenjian/cqrcb/POIRule.groovy | 3 +- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 53 +++--------- .../temp/shenjian/util/SeaweedUtils.groovy | 82 ++++++++++++++++--- 4 files changed, 95 insertions(+), 57 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy new file mode 100644 index 000000000..e499d789e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -0,0 +1,14 @@ +package com.fangjia.script.temp.shenjian + +/** + * @author shenjian* @since 2021/5/18 + */ +class Temp { + + static void main(String[] args) { + double val = 1.0 + if (val == 1.0) { + println(true) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index 5deadb989..2c5f787b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -65,6 +65,7 @@ class POIRule { //db.getCollection('gaode_address_poi').update({'name': /摄影/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': '141204', 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'typecode': /1412/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -// +//db.getCollection('gaode_address_poi').update({'type': /专科医院/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': '西南大学'}, {$set: {'status': NumberInt(0)}}, {multi:true,upsert:false}); +//db.getCollection('gaode_address_poi').update({'type': /诊所/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 66703a1f0..f910ce590 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.temp.shenjian.cqrcb import com.fangjia.core.util.DateUtil import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils @@ -86,8 +87,8 @@ class SeaweedExport { DbUtils.seaweed.find(query).limit(1).each { obj -> ret[0] = obj.developer ret[1] = obj.estate_company - ret[2] = obj.cubage_percent - ret[3] = virescencePercentValidator(obj.virescence_percent) + ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) + ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) if (obj.alias) { // 小区可能存在多个别名 @@ -98,22 +99,12 @@ class SeaweedExport { String als = temp.join(",") ret[4] = als.length() > 20 ? "" : als } - ret[5] = obj.car_site_count - if (com.fangjia.script.datastore.FieldValidatorUtils.householdCountValidator(obj.household_count) - && FieldValidatorUtils.carSiteValidator(obj.car_site_count)) { - double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) - if (val == 0.0) { - val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) - } - if (val == 0.0) { - val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(obj.car_site_count) / Double.valueOf(obj.household_count))) - } - ret[6] = "1:" + val - } + ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) + ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) ret[7] = "" - ret[8] = obj.all_area + ret[8] = SeaweedUtils.formatAllArea(obj.all_area) ret[9] = obj.loudong_count - ret[10] = obj.household_count + ret[10] = SeaweedUtils.formatHouseholdCount(obj.household_count) ret[11] = "" ret[12] = obj.estate_charge ret[13] = DateUtil.formatToYYYYMMDD(obj.open_date) @@ -133,37 +124,13 @@ class SeaweedExport { }; latch.await(); def ret = []; - ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"] + ret << ["物业公司名称","开发商名称","容积率","绿化率","楼盘别名","停车位数量","车位配比率","车位价","总占地面积","总楼栋数","总户数","非住宅总户数","物业费","开盘日期","交付时间", "经度", "纬度", "小区名称"]; for (List district : data) { def district_code = district.get(2) def name = district.get(3) ret << resultMap.get(district_code + name) - } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商物业公司.xlsx", ret, "项目统计"); + }; + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\开发商及物业信息.xlsx", ret, "开发商及物业信息"); executorService.shutdown() } - - /** - * string - * 绿化率 仅限数字,包括(小数)数字范围:10-100 - * @param virescence_percent - * @return - */ - private static def virescencePercentValidator(def virescence_percent){ - if(virescence_percent=="null"){ - return ""; - } - def value = com.fangjia.script.datastore.DistrictFieldNumberic.checkValue(virescence_percent); - if(value){ - if(value>=10 && value<=100){ - return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value)) - } else if (value > 0 && value < 1) { - return com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, Double.valueOf(value) * 100) - } else{ - return ""; - } - } - return ""; - } - } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 22969a8c8..46d7d1f0d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -1,9 +1,10 @@ package com.fangjia.script.temp.shenjian.util +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import org.apache.commons.lang.math.NumberUtils import java.util.concurrent.ConcurrentHashMap @@ -30,33 +31,33 @@ class SeaweedUtils { } return region } - + /** * 标准化输出绿化率 */ static formatVirescencePercent(def virescence_percent) { // 绿化率 - if (virescence_percent && NumberUtils.isDigits(virescence_percent)) { + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { double val = Double.valueOf(virescence_percent) if (val < 1) { return val * 100 } - if (virescence_percent == 1) { - return virescence_percent * 10 + if (val == 1) { + return val * 10 } - if (virescence_percent >= 70) { + if (val >= 70) { return 65 } - return virescence_percent + return val } return "" } /** - * 标准化绿化率输出 + * 标准化容积率输出 */ static formatCubagePercent(def cubage_percent) { - if (cubage_percent && NumberUtils.isDigits(cubage_percent)) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { double val = Double.valueOf(cubage_percent) if (val >= 7 && val < 8) { return 6.2 @@ -67,9 +68,10 @@ class SeaweedUtils { if (val >= 9 && val < 10) { return 6.5 } - if (cubage_percent && val >= 10) { + if (val >= 10) { return 6.7 } + return val } return "" } @@ -78,11 +80,12 @@ class SeaweedUtils { * 标准化车位输出 */ static formatCarSiteCount(def car_site_count) { - if (car_site_count && NumberUtils.isDigits(car_site_count)) { - long val = Long.valueOf("car_site_count") + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) if (val > 2000 || val < 100) { return "" } + return val; } return "" } @@ -91,7 +94,7 @@ class SeaweedUtils { * 标准化建筑面积输出 */ static formatBuildArea(def build_area) { - if (build_area && NumberUtils.isDigits(build_area)) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { double val = build_area if (val > 1000) { return val @@ -99,4 +102,57 @@ class SeaweedUtils { } return "" } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } } From ae5041e93f1f33e8e4ed8064bf8217a712d3051c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 18 May 2021 17:09:23 +0800 Subject: [PATCH 047/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9?= =?UTF-8?q?=E9=85=8D=E5=A5=97=E6=8E=92=E5=BA=8F=E8=BE=93=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cqrcb/AroundExportByNewWay.groovy | 21 +++++++++------- .../script/temp/shenjian/cqrcb/POIRule.groovy | 4 --- .../script/temp/shenjian/dto/AroundDTO.groovy | 25 +++++++++++++++++++ 3 files changed, 37 insertions(+), 13 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 4ec4dac97..36e9e588f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.district.Distance import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -96,10 +97,13 @@ class AroundExportByNewWay { Integer key = 1; def ret = [] for (List district : data) { + def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { - ret.addAll(resultMap.get(district_code + name)) + List currRet = resultMap.get(district_code + name) + Collections.sort(currRet) + ret.addAll(currRet) if (ret.size() > 600000) { map.put(key++, ret) ret = [] @@ -110,11 +114,10 @@ class AroundExportByNewWay { //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); for (Integer k : map.keySet()) { //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") - EasyExcel.write("F:\\数据\\重庆农商行\\周边配套"+ + k + ".xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套(文件"+ k + ")V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); } printUnMatched(unMatchedMap) executorService.shutdown() - } static calcAroundInfo(def lat, def lng, def result, def id) { @@ -130,12 +133,12 @@ class AroundExportByNewWay { if (distance == 0) { distance = 50 } - def ret = [] - ret[0] = id - ret[1] = obj.name - ret[2] = formatType(obj.typecode) - ret[3] = distance.toInteger() - result << ret + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy index 2c5f787b2..1c907e236 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/POIRule.groovy @@ -46,17 +46,13 @@ class POIRule { // 更新规则 // db.getCollection('gaode_address_poi').update({'name': /[ABCD]+栋/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含栋'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCD]+幢/, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含幢'}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[ABCD]+馆/, 'typecode': {$in: ['060102', '060101']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含馆'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDE1234567890]+区/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含区'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[ABCDEFHG]口/, 'typecode': {$nin: ['150700']}, 'status': 2}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /入口/, 'typecode': {$nin: ['150700']}, 'status': 0}, {$set: {'status': NumberInt(1), 'flag': '包含入口'}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;整形美容/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;职业技术学校/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /购物服务;商场;免税品店/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;专科医院;口腔医院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /医疗保健服务;综合医院;卫生院/}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[学]+楼/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); -//db.getCollection('gaode_address_poi').update({'name': /[教医光]+学中心/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'type': /科教文化服务;学校;学校/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /[1234]期/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); //db.getCollection('gaode_address_poi').update({'name': /住院部/, 'status': 0}, {$set: {'status': NumberInt(1)}}, {multi:true,upsert:false}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy new file mode 100644 index 000000000..39ed00ec4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dto/AroundDTO.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.dto + +/** + * @author shenjian + * @since 2021/5/18 + */ +class AroundDTO implements Comparable { + + String id; + String name; + String type + Integer distance + + @Override + public int compareTo(AroundDTO o2) { + if (this.type.equals(o2.type)) { + if (this.distance != null) { + return this.distance.compareTo(o2.distance); + } + } else { + return this.type.compareTo(o2.type); + } + return 0; + } +} From e622b12ca788f490c25e10ef521428226fdaff4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 18 May 2021 19:49:59 +0800 Subject: [PATCH 048/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=88=A4=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 1 + .../cqrcb/AroundExportByNewWay.groovy | 38 ++++++++++++------- .../temp/shenjian/cqrcb/SeaweedExport.groovy | 11 +----- .../temp/shenjian/util/SeaweedUtils.groovy | 25 ++++++++++++ 4 files changed, 52 insertions(+), 23 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e499d789e..e8a43efdb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -10,5 +10,6 @@ class Temp { if (val == 1.0) { println(true) } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 36e9e588f..5d676e7c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -96,11 +96,13 @@ class AroundExportByNewWay { Map map = new HashedMap() Integer key = 1; def ret = [] + AtomicInteger totalCount = new AtomicInteger() for (List district : data) { def district_code = district.get(2) def name = district.get(3) if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() List currRet = resultMap.get(district_code + name) Collections.sort(currRet) ret.addAll(currRet) @@ -116,6 +118,7 @@ class AroundExportByNewWay { //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") EasyExcel.write("F:\\数据\\重庆农商行\\周边配套(文件"+ k + ")V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); } + print(totalCount.get() + " " + (totalCount.get() / data.size())) printUnMatched(unMatchedMap) executorService.shutdown() } @@ -154,19 +157,28 @@ class AroundExportByNewWay { com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); } - static String formatType(String type) { - if (type.startsWith("09")) { - return "医院"; - } else if (type.startsWith('1505')) { - return "地铁"; - } else if (type.startsWith('1507')) { - return "公交"; - } else if (type.startsWith("16")) { - return "银行"; - } else if (type.startsWith("14")) { - return "学校"; - } else if (type.startsWith("06")) { - return "商场" + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index f910ce590..793ff7840 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -89,16 +89,7 @@ class SeaweedExport { ret[1] = obj.estate_company ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) - - if (obj.alias) { - // 小区可能存在多个别名 - def temp = [] - for (def a : obj.alias) { - temp.add(a) - } - String als = temp.join(",") - ret[4] = als.length() > 20 ? "" : als - } + ret[4] = SeaweedUtils.formatAlias(obj.alias) ret[5] = SeaweedUtils.formatCarSiteCount(obj.car_site_count) ret[6] = SeaweedUtils.calcCarAndHouseholdRadio(obj.car_site_count, obj.household_count) ret[7] = "" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 46d7d1f0d..2cb9a61b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -155,4 +155,29 @@ class SeaweedUtils { } return "" } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } } From f9f80f733eadee29d692895ff5ccbdab6cacfa52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 19 May 2021 11:46:43 +0800 Subject: [PATCH 049/212] =?UTF-8?q?=E5=85=AC=E4=BA=A4=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 23 +- .../cqrcb/AroundExportByNewWay.groovy | 23 +- .../shenjian/cqrcb/AroundFormatExport.groovy | 231 ++++++++++-------- .../cqrcb/CheckLatLngByDistrict.groovy | 61 +++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 8 +- 6 files changed, 233 insertions(+), 117 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e8a43efdb..8c42ab478 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,15 +1,30 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.lixue.util.DBUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - double val = 1.0 - if (val == 1.0) { - println(true) + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("region", "北碚") + query.put("cat", "district") + query.put("visible", 0) + query.put("name", "人民村") + DBUtils.seaweed.find(query).limit(1).each { obj -> + println(obj) } - } + +// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 +// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 +// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米 +// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 5d676e7c9..3bffc1300 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -59,11 +59,12 @@ class AroundExportByNewWay { query.put("cat", "district") query.put("visible", 0) query.put("name", district.get(24)) + Set set = new HashSet<>() DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> if (obj.lng && obj.lat) { def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) - calcAroundInfo(lat, lng, result, id) + calcAroundInfo(lat, lng, result, id, set) } } } else { @@ -106,7 +107,7 @@ class AroundExportByNewWay { List currRet = resultMap.get(district_code + name) Collections.sort(currRet) ret.addAll(currRet) - if (ret.size() > 600000) { + if (ret.size() > 700000) { map.put(key++, ret) ret = [] } @@ -123,7 +124,7 @@ class AroundExportByNewWay { executorService.shutdown() } - static calcAroundInfo(def lat, def lng, def result, def id) { + static calcAroundInfo(def lat, def lng, def result, def id, def set) { // 周围半径为3千米 def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) DBObject q = new BasicDBObject() @@ -136,12 +137,16 @@ class AroundExportByNewWay { if (distance == 0) { distance = 50 } - AroundDTO aroundDTO = new AroundDTO() - aroundDTO.id = id - aroundDTO.name = obj.name - aroundDTO.type = formatType(obj.typecode) - aroundDTO.distance = distance.toInteger() - result.add(aroundDTO) + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy index b445e881c..9a3f67a3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -1,31 +1,36 @@ package com.fangjia.script.temp.shenjian.cqrcb -import cn.emay.mina.util.ConcurrentHashSet -import com.fangjia.script.coredata.util.CollectionUtils -import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.alibaba.excel.EasyExcel +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger /** - * 小区周围信息格式化输出 + * 周边数据一千米配套数量统计 * * @author shenjian - * @since 2021/5/15 + * @since 2021/5/16 */ class AroundFormatExport { - static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -38,124 +43,148 @@ class AroundFormatExport { @Override void run() { try { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - def temp = [] - if (district.get(20) != null && district.get(21) != null) { - def matched_city = district.get(22) - def matched_region = district.get(23) - def matched_name = district.get(24) + List result = [] + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { DBObject query = new BasicDBObject() - query.put("matched_city", matched_city) - query.put("matched_region", matched_region) - query.put("matched_name", matched_name) - List ret = new ArrayList() - Map map = new ConcurrentHashMap<>(); - conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> - dealSpecial(obj, map, ret) + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + Set set = new HashSet<>() + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id, set) + } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - ret.add(key + ":" + (distance / count1) + "米") - } - - temp[7] = matched_city - temp[8] = matched_region - temp[9] = matched_name - if (ret.size() > 0) { - def str = ret.join(",") - temp[5] = str - println(str) - } - + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } - resultMap.put(district_code + name, temp) + resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) } catch (Exception ex) { - + ex.printStackTrace() } finally { latch.countDown() } } }) + }; + latch.await(); + def headArr = ['IDB', '数量'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) } - latch.await() + Map map = new HashedMap() + Integer key = 1; def ret = [] - ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] + AtomicInteger totalCount = new AtomicInteger() for (List district : data) { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - ret << resultMap.get(district_code + name) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() + List currRet = resultMap.get(district_code + name) + ret << [id, currRet.size()] + if (ret.size() > 700000) { + map.put(key++, ret) + ret = [] + } + } else { + ret << [id, 0] + } } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + map.put(key++, ret) + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + for (Integer k : map.keySet()) { + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + print(totalCount.get() + " " + (totalCount.get() / data.size())) + printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def ret) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false - } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false + static calcAroundInfo(def lat, def lng, def result, def id, def set) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 1000) { + if (distance == 0) { + distance = 50 + } + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) } } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - int distance = Double.valueOf(obj.distance) + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - map.put(key, child) - } else { - child.put("distance", Double.valueOf(obj.distance)) - child.put("count", 1) - map.put(key, child) - } - } else { - ret.add(obj.name + ":" + obj.distance + "米") } } + + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] + } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy new file mode 100644 index 000000000..a5338340b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger + +/** + * 通过匹配出的小区核对经纬度 + * + * @author shenjian + * @since 2021/5/19 + */ +class CheckLatLngByDistrict { + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + for (List districtT : data) { + final List district = districtT + if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + if (district.get(20) != null && district.get(21) != null) { + def originLng = district.get(20) + def originLat = district.get(21) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + if (originLng != obj.lng || originLat != obj.lat) { + def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d); + def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d); + originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d); + originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d); + + double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng) + println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" ) + } + } else { + println(district.get(24) + " 不存在经纬度") + } + } + } else { + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + + } else { + println(district.get(24) + " 不存在经纬度222222222") + } + } + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 793ff7840..86504ae9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -85,8 +85,8 @@ class SeaweedExport { filter.put("lat", 1) DbUtils.seaweed.find(query).limit(1).each { obj -> - ret[0] = obj.developer - ret[1] = obj.estate_company + ret[0] = obj.estate_company + ret[1] = obj.developer ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) ret[4] = SeaweedUtils.formatAlias(obj.alias) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 3e745dfd3..ce6475a95 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -45,7 +45,7 @@ class GaoDeUtils { }}; /** - * + * 查找周边配套 * @param city * @param kw * @param lng @@ -87,10 +87,16 @@ class GaoDeUtils { return result } + /** + * 获取经纬度之间距离 + */ static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) } + /** + * 获取地址经纬度 + */ static def parserByGaode(def city, def address) { JSONObject json = GaoDeApiUtils.geo(city, address) if (json && json.status == "1") { From 7726fa48cb6680a639242015d64c9b6a352296ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 19 May 2021 11:46:43 +0800 Subject: [PATCH 050/212] =?UTF-8?q?=E5=85=AC=E4=BA=A4=E5=8E=BB=E9=87=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 23 +- .../cqrcb/AroundExportByNewWay.groovy | 23 +- .../shenjian/cqrcb/AroundFormatExport.groovy | 252 +++++++++++------- .../cqrcb/CheckLatLngByDistrict.groovy | 61 +++++ .../temp/shenjian/cqrcb/SeaweedExport.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 8 +- 6 files changed, 254 insertions(+), 117 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e8a43efdb..8c42ab478 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,15 +1,30 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.lixue.util.DBUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - double val = 1.0 - if (val == 1.0) { - println(true) + DBObject query = new BasicDBObject() + query.put("city", "重庆") + query.put("region", "北碚") + query.put("cat", "district") + query.put("visible", 0) + query.put("name", "人民村") + DBUtils.seaweed.find(query).limit(1).each { obj -> + println(obj) } - } + +// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 +// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 +// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米 +// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米 +// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy index 5d676e7c9..3bffc1300 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExportByNewWay.groovy @@ -59,11 +59,12 @@ class AroundExportByNewWay { query.put("cat", "district") query.put("visible", 0) query.put("name", district.get(24)) + Set set = new HashSet<>() DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> if (obj.lng && obj.lat) { def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) - calcAroundInfo(lat, lng, result, id) + calcAroundInfo(lat, lng, result, id, set) } } } else { @@ -106,7 +107,7 @@ class AroundExportByNewWay { List currRet = resultMap.get(district_code + name) Collections.sort(currRet) ret.addAll(currRet) - if (ret.size() > 600000) { + if (ret.size() > 700000) { map.put(key++, ret) ret = [] } @@ -123,7 +124,7 @@ class AroundExportByNewWay { executorService.shutdown() } - static calcAroundInfo(def lat, def lng, def result, def id) { + static calcAroundInfo(def lat, def lng, def result, def id, def set) { // 周围半径为3千米 def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 3000) DBObject q = new BasicDBObject() @@ -136,12 +137,16 @@ class AroundExportByNewWay { if (distance == 0) { distance = 50 } - AroundDTO aroundDTO = new AroundDTO() - aroundDTO.id = id - aroundDTO.name = obj.name - aroundDTO.type = formatType(obj.typecode) - aroundDTO.distance = distance.toInteger() - result.add(aroundDTO) + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) + } } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy index b445e881c..89b1c7b3c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundFormatExport.groovy @@ -1,31 +1,36 @@ package com.fangjia.script.temp.shenjian.cqrcb -import cn.emay.mina.util.ConcurrentHashSet -import com.fangjia.script.coredata.util.CollectionUtils -import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.alibaba.excel.EasyExcel +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.district.Distance +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.dto.AroundDTO import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.QueryOperators +import org.apache.commons.collections.map.HashedMap import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.ConcurrentLinkedQueue import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger /** - * 小区周围信息格式化输出 + * 周边数据一千米配套数量统计 * * @author shenjian - * @since 2021/5/15 + * @since 2021/5/16 */ class AroundFormatExport { - static DBCollection conn = MongoUtils.getCollectionByName("seeddb_dev", "arround_temp", "192.168.0.235", 27018) + static DBCollection poiConn = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) static def resultMap = new ConcurrentHashMap() + static def unMatchedMap = new ConcurrentHashMap() static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { @@ -38,124 +43,169 @@ class AroundFormatExport { @Override void run() { try { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - def temp = [] - if (district.get(20) != null && district.get(21) != null) { - def matched_city = district.get(22) - def matched_region = district.get(23) - def matched_name = district.get(24) + List result = [] + if (district.size() > 21 && district.get(20) != null && district.get(21) != null) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(20) / 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, district.get(21) / 10000000000d) + calcAroundInfo(lat, lng, result, id) + } else if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { DBObject query = new BasicDBObject() - query.put("matched_city", matched_city) - query.put("matched_region", matched_region) - query.put("matched_name", matched_name) - List ret = new ArrayList() - Map map = new ConcurrentHashMap<>(); - conn.find(query, new BasicDBObject("name": 1, "distance": 1, "matched_region": 1)).batchSize(1000).each { obj -> - dealSpecial(obj, map, ret) + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + Set set = new HashSet<>() + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each {obj -> + if (obj.lng && obj.lat) { + def lng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng/ 10000000000d) + def lat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d) + calcAroundInfo(lat, lng, result, id, set) + } } - for (String key : map.keySet()) { - int distance = map.get(key).get("distance") - int count1 = map.get(key).get("count") - ret.add(key + ":" + (distance / count1) + "米") - } - - temp[7] = matched_city - temp[8] = matched_region - temp[9] = matched_name - if (ret.size() > 0) { - def str = ret.join(",") - temp[5] = str - println(str) - } - + } else { + def ret = [] + ret[0] = id + ret[1] = district.get(1) + ret[2] = district_code + ret[3] = name + ret[4] = district.get(4) + unMatchedMap.put(id, ret) } - resultMap.put(district_code + name, temp) + resultMap.put(district_code + name, result) println("当前处理" + count.incrementAndGet() ) } catch (Exception ex) { - + ex.printStackTrace() } finally { latch.countDown() } } }) + }; + latch.await(); + def headArr = ['IDB', '数量'] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) } - latch.await() + Map map = new HashedMap() + Integer key = 1; def ret = [] - ret << ['IDB','城市名称','区域编码','小区名称','楼栋号','3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息','1km范围内地铁站、公交站、银行、医院、商场、学校等数量'] + AtomicInteger totalCount = new AtomicInteger() for (List district : data) { + def id = district.get(0) def district_code = district.get(2) def name = district.get(3) - ret << resultMap.get(district_code + name) + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(resultMap.get(district_code + name))) { + totalCount.incrementAndGet() + List currRet = resultMap.get(district_code + name) + int gj = 0 + int dt = 0 + int yh = 0 + int yy = 0 + int sc = 0 + int xx = 0 + for (AroundDTO dto : currRet) { + if(dto.getType() == '公交') { + gj++ + } else if(dto.getType() == '地铁') { + dt++ + } else if(dto.getType() == '银行') { + yh++ + } else if (dto.getType() == '医院') { + yy++ + } else if(dto.getType() == '商场') { + sc++ + } else if(dto.getType() == '学校') { + xx++ + } + } + ret << [id, "公交:" + gj + ";" + "地铁:" + dt + ";银行:" + yh + ";医院:" + yy + ";商场:" + sc + ";学校:" + xx] + if (ret.size() > 700000) { + map.put(key++, ret) + ret = [] + } + } else { + ret << [id, ""] + } } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + map.put(key++, ret) + //com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套.xlsx", ret, "周边配套"); + for (Integer k : map.keySet()) { + //FileUtils.batchCreateFile(map.get(k), "F:\\数据\\重庆农商行\\周边配套"+ k + ".dat", headArr, "utf-8") + EasyExcel.write("F:\\数据\\重庆农商行\\周边配套1千米V1.xlsx").head(head).sheet("周边配套").doWrite(map.get(k)); + } + print(totalCount.get() + " " + (totalCount.get() / data.size())) + printUnMatched(unMatchedMap) executorService.shutdown() } - static dealSpecial(DBObject obj, def map, def ret) { - def name = obj.name - if (name.indexOf("/") != -1) { - return false - } - if (name == "1号楼") { - return false - } - if (name.indexOf("地铁口") == -1 && name.indexOf("公交站") == -1 && name.indexOf("支行") == -1) { - if (name.indexOf("医院") != -1 && !name.endsWith("医院")) { - if (name.indexOf("分院") == -1) { - return false + static calcAroundInfo(def lat, def lng, def result, def id, def set) { + // 周围半径为3千米 + def mapLatLng = CommonDatautils.findMaxLatLngMinLatLng(lat, lng, 1000) + DBObject q = new BasicDBObject() + q.put('status', 0) + q.put('lat', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLat).append(QueryOperators.GTE, mapLatLng.minLat)); + q.put('lng', new BasicDBObject(QueryOperators.LTE, mapLatLng.maxLng).append(QueryOperators.GTE, mapLatLng.minLng)); + poiConn.find(q, new BasicDBObject("name": 1, "distance": 1, "type": 1, "typecode": 1, "lat": 1, "lng": 1)).batchSize(500).each {obj -> + double distance = Distance.GetDistance(lng, lat, obj.lng, obj.lat) + if (distance <= 1000) { + if (distance == 0) { + distance = 50 + } + if (!set.contains(obj.name)) { + AroundDTO aroundDTO = new AroundDTO() + aroundDTO.id = id + aroundDTO.name = obj.name + aroundDTO.type = formatType(obj.typecode) + aroundDTO.distance = distance.toInteger() + result.add(aroundDTO) + + set.add(obj.name) } } - if (name.indexOf("卫生中心") != -1 && !name.endsWith("卫生中心")) { - return false - } - if (name.indexOf("大学") != -1 && !name.endsWith("大学")) { - if (name.indexOf("校区") == -1 && name.indexOf("幼儿园") == -1 && name.indexOf("小学") == -1 && name.indexOf("中学") == -1 - && !name.endsWith("医院")) { - return false - } - } - if (name.indexOf("学院") != -1 && !name.endsWith("学院")) { - return false - } - if (name.indexOf("分院") != -1 && !name.endsWith("分院")) { - return false - } - if (name.indexOf("学校") != -1 && !name.endsWith("学校")) { - if (name.indexOf("校区") == -1 && name.indexOf("分校") == -1) { - return false - } - } - if (name.indexOf("中学") != -1 && !name.endsWith("中学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - if (name.indexOf("小学") != -1 && !name.endsWith("小学")) { - if (name.indexOf("校区") == -1) { - return false - } - } - } - // 中梁山地铁站1A口 - if (obj.name.indexOf("地铁站") != -1 && obj.name.indexOf("口") != -1) { - def nameTemp = obj.name.substring(0, obj.name.indexOf("地铁站") + 3) - def key = nameTemp - Map child = new HashMap() - if (map.get(key) != null) { - int distance = Double.valueOf(obj.distance) + map.get(key).get("distance") - int count1 = map.get(key).get("count") - child.put("distance", distance) - child.put("count", count1) - map.put(key, child) - } else { - child.put("distance", Double.valueOf(obj.distance)) - child.put("count", 1) - map.put(key, child) - } - } else { - ret.add(obj.name + ":" + obj.distance + "米") } } + + static printUnMatched(def unMatchedMap) { + def ret = [] + ret << ['IDB','城市名称','区域编码','小区名称','楼栋号', '经度', '纬度'] + + for (def r :unMatchedMap.values()) { + ret << r; + } + + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\重庆农商行\\周边配套未匹配.xlsx", ret, "周边配套"); + } + + static String formatType(String typeStr) { + def types = [] + if (typeStr.indexOf("|") > -1) { + types = typeStr.split("\\|") + } else { + types =[typeStr] + } + for (def type : types) { + if (type.startsWith("09")) { + return "医院"; + } else if (type.startsWith('1505')) { + return "地铁"; + } else if (type.startsWith('1507')) { + return "公交"; + } else if (type.startsWith("16")) { + return "银行"; + } else if (type.startsWith("14")) { + return "学校"; + } else if (type.startsWith("06")) { + return "商场" + } + } + return "未知" + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy new file mode 100644 index 000000000..a5338340b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/CheckLatLngByDistrict.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.temp.shenjian.cqrcb + +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.atomic.AtomicInteger + +/** + * 通过匹配出的小区核对经纬度 + * + * @author shenjian + * @since 2021/5/19 + */ +class CheckLatLngByDistrict { + + static void main(String[] args) { + List> data = ExcelUtils.read("F:\\数据\\重庆农商行\\测试数据汇总_2021.xlsx", 9) + for (List districtT : data) { + final List district = districtT + if (district.size() >24 &&district.get(22) != null && district.get(23) != null && district.get(24) != null) { + DBObject query = new BasicDBObject() + query.put("city", district.get(22)) + query.put("region", district.get(23)) + query.put("cat", "district") + query.put("visible", 0) + query.put("name", district.get(24)) + if (district.get(20) != null && district.get(21) != null) { + def originLng = district.get(20) + def originLat = district.get(21) + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + if (originLng != obj.lng || originLat != obj.lat) { + def nowLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lat / 10000000000d); + def nowLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, obj.lng / 10000000000d); + originLng = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLng / 10000000000d); + originLat = com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, originLat / 10000000000d); + + double distance = GaoDeUtils.getDistance(nowLat, nowLng, originLat,originLng) + println(district.get(24) + " 经度纬度不同" + nowLng + "," + nowLat + " 原始" + originLng + "," + originLat + " 距离" + distance + "米" ) + } + } else { + println(district.get(24) + " 不存在经纬度") + } + } + } else { + DBUtils.seaweed.find(query, new BasicDBObject("lng", 1).append("lat", 1)).limit(1).each { obj -> + if (obj.lng && obj.lat) { + + } else { + println(district.get(24) + " 不存在经纬度222222222") + } + } + } + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy index 793ff7840..86504ae9a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SeaweedExport.groovy @@ -85,8 +85,8 @@ class SeaweedExport { filter.put("lat", 1) DbUtils.seaweed.find(query).limit(1).each { obj -> - ret[0] = obj.developer - ret[1] = obj.estate_company + ret[0] = obj.estate_company + ret[1] = obj.developer ret[2] = SeaweedUtils.formatCubagePercent(obj.cubage_percent) ret[3] = SeaweedUtils.formatVirescencePercent(obj.virescence_percent) ret[4] = SeaweedUtils.formatAlias(obj.alias) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 3e745dfd3..ce6475a95 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -45,7 +45,7 @@ class GaoDeUtils { }}; /** - * + * 查找周边配套 * @param city * @param kw * @param lng @@ -87,10 +87,16 @@ class GaoDeUtils { return result } + /** + * 获取经纬度之间距离 + */ static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) } + /** + * 获取地址经纬度 + */ static def parserByGaode(def city, def address) { JSONObject json = GaoDeApiUtils.geo(city, address) if (json && json.status == "1") { From a00200c12e78977bf322cb4e3a9ab9794c6f012b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 11:22:13 +0800 Subject: [PATCH 051/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9POI?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/cqrcb/AllExport.groovy | 2 +- .../temp/shenjian/cqrcb/AroundExport.groovy | 2 +- .../script/temp/shenjian/cqrcb/SavePoi.groovy | 2 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 124 ++++++++++++++++++ .../fangjia/script/temp/shenjian/poi/poi.json | 38 ++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 70 +++++++++- .../temp/shenjian/util/HttpUtils.groovy | 29 ++++ .../temp/shenjian/util/SeaweedUtils.groovy | 8 ++ .../wenyan/auto/KeywordSearchUtils.groovy | 1 + .../script/statistics/poi/AmapPoi.java | 71 ++++++++++ 10 files changed, 338 insertions(+), 9 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy index 99bf24d62..0ed85bb0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AllExport.groovy @@ -27,7 +27,7 @@ class AllExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy index fd3450f87..28cec6534 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/AroundExport.groovy @@ -27,7 +27,7 @@ class AroundExport { //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 //static ExecutorService executorService = Executors.newFixedThreadPool(30) // 高德地图中不存在周边的数据 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy index 9da03d468..40c42f36a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/cqrcb/SavePoi.groovy @@ -21,7 +21,7 @@ class SavePoi { static DBCollection conn43 = MongoUtils.getCollectionByName("address_analytics", "gaode_address_poi", "192.168.0.43", 28017) //3km范围内地铁站、公交站、银行、医院、商场、学校等名称及距离信息 static def types = "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|0601|141200|141201|141202|141203|141204" - static def radius = "3000" + static def radius = 3000 static ExecutorService executorService = Executors.newFixedThreadPool(30) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..9b2599516 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,124 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoi { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(5000).each {obj -> + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json new file mode 100644 index 000000000..4777a3761 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json @@ -0,0 +1,38 @@ +{ + "id": "B0FFF9RJ2O", + "parent": [], + "childtype": [], + "name": "丁兰广场", + "type": "购物服务;商场;购物中心", + "typecode": "060101", + "biz_type": [], + "address": "惠兰雅路与明珠街交汇处东北角", + "location": "120.231368,30.353127", + "tel": "0571-88111223", + "pname": "浙江省", + "cityname": "杭州市", + "adname": "上城区", + "importance": [], + "shopid": [], + "shopinfo": "0", + "poiweight": [], + "distance": "37", + "biz_ext": { + "rating": [], + "cost": "13953.00" + }, + "photos": [ + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/f5dcc118938358031b45dea7e04faf45" + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/9a0fd6ceec63e030eac3ee14055206b5" + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/d9d2a87de3f00294665a2caeb3ee226f" + } + ] +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index ce6475a95..f772c3ab8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -1,12 +1,12 @@ package com.fangjia.script.temp.shenjian.util +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import net.sf.json.JSONObject - -import java.util.concurrent.ConcurrentHashMap /** * 高德工具类 @@ -16,7 +16,8 @@ import java.util.concurrent.ConcurrentHashMap */ class GaoDeUtils { - def static GAO_DE_AK = "2a1fb75990f2b39f6221e50892dd3988" + // 新获取key + def static GAO_DE_AK = "4e9592208336433ee1b7aa4d1d434418" def static codeNameMap = new HashMap(){{ put("150500", "地铁"); @@ -54,7 +55,7 @@ class GaoDeUtils { * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) * @return */ - static List searchNearByPoi(String city, String types, Double lng, Double lat, String radius, String keywords){ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] params.key = GAO_DE_AK @@ -64,7 +65,7 @@ class GaoDeUtils { params.keywords = keywords } params.city = city - params.radius = radius + params.radius = String.valueOf(radius) params.location = "${lng},${lat}".toString() List result = [] @@ -80,6 +81,63 @@ class GaoDeUtils { if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + JSONArray jsonArray = json.pois + List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); + result.addAll(ret) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; } else { break } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 2cb9a61b2..22673723b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -176,6 +176,14 @@ class SeaweedUtils { return "" } + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + static void main(String[] args) { def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] println(formatAlias(alias)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy index a2026322b..c3a0c4acf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy @@ -204,6 +204,7 @@ public class KeywordSearchUtils{ put("旅游景点","110000"); put("体育场馆","0800|0801"); put("图书馆","140500"); + put("住宅区","120300|120301|120302|120303|120304"); }}; diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From d81f24748874c32ea2c9d4682dc1a99ed8a563a2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 14:06:16 +0800 Subject: [PATCH 052/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E7=BF=BB=E9=A1=B5?= =?UTF-8?q?=E5=90=8E=E9=95=BF=E5=BA=A6=E4=B8=BA1=E8=BD=ACjsonArray?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 16 ++-- .../fangjia/script/temp/shenjian/poi/poi.json | 84 +++++++++++-------- .../temp/shenjian/util/GaoDeUtils.groovy | 9 +- 3 files changed, 60 insertions(+), 49 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 8c42ab478..8311e3e15 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,6 +1,9 @@ package com.fangjia.script.temp.shenjian +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi import com.mongodb.BasicDBObject import com.mongodb.DBObject @@ -10,15 +13,10 @@ import com.mongodb.DBObject class Temp { static void main(String[] args) { - DBObject query = new BasicDBObject() - query.put("city", "重庆") - query.put("region", "北碚") - query.put("cat", "district") - query.put("visible", 0) - query.put("name", "人民村") - DBUtils.seaweed.find(query).limit(1).each { obj -> - println(obj) - } + String jsonStr = "[{\"id\":\"B0FFLCY016\",\"parent\":[],\"childtype\":[],\"name\":\"杭州馨悦坊公寓\",\"type\":\"住宿服务;住宿服务相关;住宿服务相关\",\"typecode\":\"100000\",\"biz_type\":\"hotel\",\"address\":\"18号大街伊萨卡浩泽园\",\"location\":\"120.383947,30.289931\",\"tel\":[],\"pname\":\"浙江省\",\"cityname\":\"杭州市\",\"adname\":\"钱塘区\",\"importance\":[],\"shopid\":[],\"shopinfo\":\"0\",\"poiweight\":[],\"distance\":\"992\",\"biz_ext\":{\"rating\":[],\"cost\":[],\"hotel_ordering\":\"0\",\"lowest_price\":[],\"star\":[],\"open_time\":[],\"opentime2\":[]},\"photos\":[{\"title\":\"Logo\",\"url\":\"http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c\",\"provider\":[]},{\"title\":\"客房\",\"url\":\"http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74\",\"provider\":[]},{\"title\":[],\"url\":\"http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315\",\"provider\":[]}]}]" + JSONArray jsonArray = JSONObject.parseArray(jsonStr) + List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); + print(ret.size()) } // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json index 4777a3761..2cc6544cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/poi.json @@ -1,38 +1,48 @@ -{ - "id": "B0FFF9RJ2O", - "parent": [], - "childtype": [], - "name": "丁兰广场", - "type": "购物服务;商场;购物中心", - "typecode": "060101", - "biz_type": [], - "address": "惠兰雅路与明珠街交汇处东北角", - "location": "120.231368,30.353127", - "tel": "0571-88111223", - "pname": "浙江省", - "cityname": "杭州市", - "adname": "上城区", - "importance": [], - "shopid": [], - "shopinfo": "0", - "poiweight": [], - "distance": "37", - "biz_ext": { - "rating": [], - "cost": "13953.00" - }, - "photos": [ - { - "title": [], - "url": "http://store.is.autonavi.com/showpic/f5dcc118938358031b45dea7e04faf45" +[ + { + "id": "B0FFLCY016", + "parent": [], + "childtype": [], + "name": "杭州馨悦坊公寓", + "type": "住宿服务;住宿服务相关;住宿服务相关", + "typecode": "100000", + "biz_type": "hotel", + "address": "18号大街伊萨卡浩泽园", + "location": "120.383947,30.289931", + "tel": [], + "pname": "浙江省", + "cityname": "杭州市", + "adname": "钱塘区", + "importance": [], + "shopid": [], + "shopinfo": "0", + "poiweight": [], + "distance": "992", + "biz_ext": { + "rating": [], + "cost": [], + "hotel_ordering": "0", + "lowest_price": [], + "star": [], + "open_time": [], + "opentime2": [] }, - { - "title": [], - "url": "http://store.is.autonavi.com/showpic/9a0fd6ceec63e030eac3ee14055206b5" - }, - { - "title": [], - "url": "http://store.is.autonavi.com/showpic/d9d2a87de3f00294665a2caeb3ee226f" - } - ] -} \ No newline at end of file + "photos": [ + { + "title": "Logo", + "url": "http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c", + "provider": [] + }, + { + "title": "客房", + "url": "http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74", + "provider": [] + }, + { + "title": [], + "url": "http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315", + "provider": [] + } + ] + } +] \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index f772c3ab8..9f277a4b2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -129,9 +129,12 @@ class GaoDeUtils { } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() - JSONArray jsonArray = json.pois - List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); - result.addAll(ret) + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } } else if (json && json.infocode && json.infocode == 10003) { throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") } else if (json && json.infocode && json.infocode == 10004) { From 9f5d0ca222cf4ff53f92862fc3849acb97a3e09d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 14:19:29 +0800 Subject: [PATCH 053/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=A4=9A=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E6=96=B9=E5=BC=8F=E8=B0=83=E7=94=A8=E5=91=A8=E8=BE=B9?= =?UTF-8?q?POI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 37 ++++++++++++------- .../temp/shenjian/util/GaoDeUtils.groovy | 8 ++-- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 9b2599516..28ac4a0ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -16,6 +16,9 @@ import com.mongodb.DBObject import org.apache.commons.collections.CollectionUtils import org.bson.types.ObjectId +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + /** * 小区周边POI数据获取 * @@ -26,26 +29,34 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { def types = listAllType() //def types = "0601" DBObject query = new BasicDBObject() query.put("status", 1) - conn136.find(query).batchSize(5000).each {obj -> - def city = obj.city - def lng = obj.lng - def lat = obj.lat - List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") - if (CollectionUtils.isNotEmpty(pois)) { - String jsonStr = JSONObject.toJSONString(pois) - HttpUtils.postForJson(URL, jsonStr) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) - } else { - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) - } - println("已处理小区:" + obj) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) } + executorService.shutdown() } static getNewSeaweed() { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 9f277a4b2..e384c35fb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -135,9 +135,11 @@ class GaoDeUtils { } catch(Exception ex) { ex.printStackTrace() } - } else if (json && json.infocode && json.infocode == 10003) { - throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") - } else if (json && json.infocode && json.infocode == 10004) { + } else if (json && json.infocode && json.infocode == "10044") { + //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + // 多线程情况下还是exit有效果,抛异常不会停止程序 + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; From bf14e84e49198979efecc3bbb6b884d866d5866d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 15:52:01 +0800 Subject: [PATCH 054/212] =?UTF-8?q?=E6=9A=82=E6=97=B6=E5=B1=8F=E8=94=BD?= =?UTF-8?q?=E4=B8=8D=E9=87=8D=E8=A6=81=E7=B1=BB=E5=9E=8B=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../wenyan/auto/KeywordSearchUtils.groovy | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy index c3a0c4acf..278aced74 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/auto/KeywordSearchUtils.groovy @@ -179,32 +179,33 @@ public class KeywordSearchUtils{ public static final Map KEYWORDS_MAP = new HashMap(){{ put("地铁", "150500"); put("公交", "1507"); - put("码头", "1503"); - put("机场", "150104|150106"); - put("火车站", "150200"); put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); put("写字楼", "120201"); - put("停车场", "150903|150904|150905|150906"); - put("网吧", "080308"); put("学校","141200|141201|141202|141203|141204"); put("银行","1601|1603"); - put("公园/绿地","110100|110101"); - put("KTV","080302"); - put("健身中心","080111"); put("大型超市","0604"); - put("咖啡厅","0505"); - put("剧场/影院","080601|080603"); put("商场","0601"); - put("酒店","10"); - put("整容整形","090201"); - put("洗浴中兴","0714"); - put("便利店","0602"); put("餐饮","0501|0502|0503|0504"); - put("游乐场","080501"); put("旅游景点","110000"); - put("体育场馆","0800|0801"); put("图书馆","140500"); - put("住宅区","120300|120301|120302|120303|120304"); + +// put("码头", "1503");1 +// put("机场", "150104|150106");1 +// put("火车站", "150200");1 +// put("停车场", "150903|150904|150905|150906");1 +// put("网吧", "080308");1 +// put("公园/绿地","110100|110101"); +// put("KTV","080302");1 +// put("健身中心","080111");1 +// put("咖啡厅","0505");1 +// put("剧场/影院","080601|080603");1 +// put("酒店","10");1 +// put("整容整形","090201");1 +// put("洗浴中兴","0714");1 +// put("便利店","0602");1 +// put("游乐场","080501");1 +// put("体育场馆","0800|0801");1 +// put("住宅区","120300|120301|120302|120303|120304"); }}; From 387f6ec98b8f0cbe362afbde21c5d88d140bddb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 18:01:23 +0800 Subject: [PATCH 055/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BAgeo?= =?UTF-8?q?hash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 14 +++--- .../seaweed/SeaweedGeoHashExport.groovy | 44 +++++++++++++++++++ .../script/temp/shenjian/seaweed/Temp.groovy | 28 ------------ .../temp/shenjian/util/SeaweedUtils.groovy | 17 +++++++ 4 files changed, 68 insertions(+), 35 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 8311e3e15..e95cbadff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -2,10 +2,9 @@ package com.fangjia.script.temp.shenjian import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject -import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.statistics.poi.AmapPoi -import com.mongodb.BasicDBObject -import com.mongodb.DBObject +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import org.elasticsearch.geometry.utils.Geohash /** * @author shenjian* @since 2021/5/18 @@ -13,10 +12,11 @@ import com.mongodb.DBObject class Temp { static void main(String[] args) { - String jsonStr = "[{\"id\":\"B0FFLCY016\",\"parent\":[],\"childtype\":[],\"name\":\"杭州馨悦坊公寓\",\"type\":\"住宿服务;住宿服务相关;住宿服务相关\",\"typecode\":\"100000\",\"biz_type\":\"hotel\",\"address\":\"18号大街伊萨卡浩泽园\",\"location\":\"120.383947,30.289931\",\"tel\":[],\"pname\":\"浙江省\",\"cityname\":\"杭州市\",\"adname\":\"钱塘区\",\"importance\":[],\"shopid\":[],\"shopinfo\":\"0\",\"poiweight\":[],\"distance\":\"992\",\"biz_ext\":{\"rating\":[],\"cost\":[],\"hotel_ordering\":\"0\",\"lowest_price\":[],\"star\":[],\"open_time\":[],\"opentime2\":[]},\"photos\":[{\"title\":\"Logo\",\"url\":\"http://store.is.autonavi.com/showpic/7e9c5bdeeba929eb4742102b7c9bd47c\",\"provider\":[]},{\"title\":\"客房\",\"url\":\"http://store.is.autonavi.com/showpic/e4e5771e4de5083877931fda2e360f74\",\"provider\":[]},{\"title\":[],\"url\":\"http://store.is.autonavi.com/showpic/5401e1e105bcbacf550896c7575ee315\",\"provider\":[]}]}]" - JSONArray jsonArray = JSONObject.parseArray(jsonStr) - List ret = JSONObject.parseArray(jsonArray.toJSONString(), AmapPoi.class); - print(ret.size()) + String str = SeaweedUtils.encodeByGeoHash(106.569161, 29.498502) + println(str) + str = Geohash.stringEncode(106.569161, 29.498502, 10) + println(str) + // wm5zc4131rmf } // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy new file mode 100644 index 000000000..dad81abe4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区geohash导出 + * + * @author shenjian + * @since 2021/5/25 + */ +class SeaweedGeoHashExport { + + static void main(String[] args) { + Set set = new HashSet<>() + + DBObject query = new BasicDBObject() + query.append("cat", "district").append("visible", 0) + + DBObject filter = new BasicDBObject() + filter.put("lng", 1) + filter.put("lat", 1) + DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp -> + final def obj = objTemp + if (obj.lng && obj.lat) { + def lon = obj.lng + def lat = obj.lat + set.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) + } + } + def ret = [] + ret << ['小区geohash'] + for (String val : set) { + ret << [val] + } + com.fangjia.script.util.ExcelUtils.write("F:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy deleted file mode 100644 index d85ddf81f..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/Temp.groovy +++ /dev/null @@ -1,28 +0,0 @@ -package com.fangjia.script.temp.shenjian.seaweed - -import com.fangjia.script.util.DbUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBObject - -class Temp { - - static main (args){ - DBObject query = new BasicDBObject() - query.append("cat", "district").append("visible", 0).append("city", "北京") - - DBObject filter = new BasicDBObject() - filter.append("build_structure", 1) - - Set set = new HashSet<>() - - DbUtils.seaweed.find(query, filter).batchSize(10000).each({ dbo -> - if (dbo.build_structure) { - set.add(dbo.build_structure) - } - }) - - set.each {val -> - println(val) - } - } -} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 22673723b..9fa4739f8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -5,6 +5,7 @@ import com.fangjia.script.datastore.FieldValidatorUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash import java.util.concurrent.ConcurrentHashMap @@ -184,6 +185,22 @@ class SeaweedUtils { return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) } + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + static void main(String[] args) { def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] println(formatAlias(alias)) From 0aa41d570a5b31f0450ae3a0f45c776f3ae1a3d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 25 May 2021 18:15:35 +0800 Subject: [PATCH 056/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BAgeo?= =?UTF-8?q?hash?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/SeaweedGeoHashExport.groovy | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy index dad81abe4..154efe43e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy @@ -5,10 +5,6 @@ import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject -import java.util.concurrent.CountDownLatch -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors - /** * 小区geohash导出 * @@ -18,10 +14,10 @@ import java.util.concurrent.Executors class SeaweedGeoHashExport { static void main(String[] args) { - Set set = new HashSet<>() + List list = new ArrayList<>() DBObject query = new BasicDBObject() - query.append("cat", "district").append("visible", 0) + query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁") DBObject filter = new BasicDBObject() filter.put("lng", 1) @@ -31,12 +27,12 @@ class SeaweedGeoHashExport { if (obj.lng && obj.lat) { def lon = obj.lng def lat = obj.lat - set.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) + list.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) } } def ret = [] ret << ['小区geohash'] - for (String val : set) { + for (String val : list) { ret << [val] } com.fangjia.script.util.ExcelUtils.write("F:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); From 5ccb7e5e212d24639996e4510cba707b0d4eee3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 11:57:30 +0800 Subject: [PATCH 057/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=80=BB=E9=87=8F=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/{ => 58}/City58.groovy | 0 .../shenjian/crawl/{ => 58}/Comm58.groovy | 0 .../shenjian/crawl/58/Count58Seaweed.groovy | 49 +++++++ .../shenjian/crawl/{ => 58}/Seed58.groovy | 0 .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 2 +- .../poi/SeaweedAroundPoiSingal.groovy | 128 ++++++++++++++++++ 6 files changed, 178 insertions(+), 1 deletion(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/City58.groovy (100%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Comm58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Seed58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy new file mode 100644 index 000000000..0c9ea6f30 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy @@ -0,0 +1,49 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.crawl.InvokeHttpRequest +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * 统计58小区总量 + * + * @author shenjian + * @since 2021/5/26 + */ +class Count58Seaweed { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { obj -> + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".58.com/xiaoqu/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + String str = JQueryUtils.selectText(".total-info", doc) + // 117.60.236.41 + if (str) { + str = str.replace("共找到", "").replace("个小区", "").trim() + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count))) + println("当前处理城市" + obj.city) + } + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 28ac4a0ab..016d7be75 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,7 +29,7 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { def types = listAllType() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy new file mode 100644 index 000000000..be70b79e5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -0,0 +1,128 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取单线程方式 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiSingal { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(30).each {objTemp -> + final def obj = objTemp + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} From bc741734477f00b27196a8bd56c212d0fafc04d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 11:57:30 +0800 Subject: [PATCH 058/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=80=BB=E9=87=8F=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/{ => 58}/City58.groovy | 0 .../shenjian/crawl/{ => 58}/Comm58.groovy | 0 .../shenjian/crawl/58/Count58Seaweed.groovy | 65 +++++++++ .../shenjian/crawl/{ => 58}/Seed58.groovy | 0 .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 2 +- .../poi/SeaweedAroundPoiSingal.groovy | 128 ++++++++++++++++++ 6 files changed, 194 insertions(+), 1 deletion(-) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/City58.groovy (100%) rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Comm58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy rename projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/{ => 58}/Seed58.groovy (100%) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/City58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Comm58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Comm58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy new file mode 100644 index 000000000..ca8e4379c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Count58Seaweed.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.crawl.InvokeHttpRequest +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +/** + * 统计58小区总量 + * + * @author shenjian + * @since 2021/5/26 + */ +class Count58Seaweed { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + + static void main(String[] args) { + crawlSeaweedCount() + println(count()) + } + + static crawlSeaweedCount() { + cityConn.find(new BasicDBObject("root", "58.com").append("district_count", new BasicDBObject('$eq', null)), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { obj -> + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".58.com/xiaoqu/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + String str = JQueryUtils.selectText(".total-info", doc) + // 117.60.236.41 + if (str) { + str = str.replace("共找到", "").replace("个小区", "").trim() + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("district_count", count))) + println("当前处理城市" + obj.city) + } + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + } + + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "58.com"), new BasicDBObject("district_count", 1)) + .batchSize(100).each { obj -> + if (obj.district_count) { + number += obj.district_count + } + } + return number + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy similarity index 100% rename from projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/Seed58.groovy rename to projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/Seed58.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 28ac4a0ab..016d7be75 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,7 +29,7 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { def types = listAllType() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy new file mode 100644 index 000000000..be70b79e5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -0,0 +1,128 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取单线程方式 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundPoiSingal { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + def types = listAllType() + //def types = "0601" + DBObject query = new BasicDBObject() + query.put("status", 1) + conn136.find(query).batchSize(30).each {objTemp -> + final def obj = objTemp + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } +} From 7fe4a709f5d848225d9474fee6e5cc7a6902cd8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:16:14 +0800 Subject: [PATCH 059/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 5 ++ .../crawl/58/DistrictDetailCrawl.groovy | 46 +++++++++++++++++++ .../temp/shenjian/util/MessageUtils.groovy | 45 ++++++++++++++++++ .../fangjia/script/util/CommonUtils.groovy | 11 ++++- 4 files changed, 106 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 0d5adfa99..871548bdf 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -487,5 +487,10 @@ 1.18.20 provided + + org.msgpack + msgpack + 0.6.12 + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy new file mode 100644 index 000000000..095456eed --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +/** + * 58小区详情爬取 + * + * @author shenjian + * @since 2021/5/26 + */ +class DistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { obj -> + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String)content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + println("已抓取" + obj.district_id) + } + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy new file mode 100644 index 000000000..93174d945 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MessageUtils.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSON +import org.msgpack.MessagePack +import org.msgpack.type.Value; + +/** + * msg解析工具类 + */ +public class MessageUtils { + + public static String unpack(byte[] content) { + MessagePack messagePack = new MessagePack(); + try { + Value value = messagePack.read(content); + return value.toString(); + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + + public static byte[] pack(String value){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(value) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } + + public static byte[] pack(JSON json){ + MessagePack messagePack = new MessagePack(); + try { + byte[] data = messagePack.write(json) + return data; + } catch (IOException e) { + e.printStackTrace(); + } + return null; + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy index 3d1d74c0c..d37680fac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/CommonUtils.groovy @@ -167,7 +167,16 @@ public class CommonUtils { } - + static void sendMessage(String queueName, byte[] bytes){ + ConnectionFactory factory = new ConnectionFactory(); + factory.setUri("amqp://guest:guest@192.168.0.19:45672/%2f"); + Connection conn = factory.newConnection(); + Channel channel = conn.createChannel(); + channel.queueDeclare(queueName, true, false, false, null); + channel.basicPublish("", queueName, null, bytes); + channel.close() + conn.close() + } /** From 019a18b28e30cc0a4bd6990b3c097238cd1bcc60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:17:52 +0800 Subject: [PATCH 060/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../crawl/58/District58DetailCrawl.groovy | 67 +++++++++++++++++++ .../crawl/58/DistrictDetailCrawl.groovy | 46 ------------- 2 files changed, 67 insertions(+), 46 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..ca7a1b7bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,67 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +/** + * 58小区详情爬取 + * + * @author shenjian + * @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + + static void main(String[] args) { + StopWatch stopWatch = new StopWatch(); + stopWatch.start("District58DetailCrawl") + long count = 1; + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { obj -> + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String)content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + if (count % 20 == 0) { + stopWatch.stop() + log.info("抓取总量为{}, 耗时{}", count, stopWatch.getTotalTimeSeconds() / 60) + stopWatch.start() + } + count++ + log.info("已抓取 {}", obj.district_id) + } + } + } + } + if (stopWatch.isRunning()) { + stopWatch.stop() + log.info("抓取完毕。。。抓取总量为{}, 耗时{}", count++, stopWatch.getTotalTimeSeconds() / 60) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy deleted file mode 100644 index 095456eed..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy +++ /dev/null @@ -1,46 +0,0 @@ -package com.fangjia.script.temp.shenjian.crawl - -import com.alibaba.fastjson.JSON -import com.alibaba.fastjson.JSONObject -import com.fangjia.script.temp.shenjian.util.MessageUtils -import com.fangjia.script.util.ABuYunUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection - -/** - * 58小区详情爬取 - * - * @author shenjian - * @since 2021/5/26 - */ -class DistrictDetailCrawl { - - static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) - - static void main(String[] args) { - districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) - .batchSize(1000).limit(1000).each { obj -> - final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" - def res = ABuYunUtil.getReponseContent2(URL, "utf-8") - if (res && res.code == 200) { - String content = res.content - - JSON json = new JSONObject() - json.put("project", "district_detail_58") - json.put("url", URL) - - JSON result = new JSONObject() - result.put("content", (String)content) - json.put("result", result) - - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) - districtSeedConn.update(new BasicDBObject("_id", obj._id), - new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) - println("已抓取" + obj.district_id) - } - } - } -} \ No newline at end of file From bfbcdd07fa9c1cfa6115b4eb5f4e733166c9c81a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 26 May 2021 17:17:52 +0800 Subject: [PATCH 061/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E58=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC=E6=8A=93=E5=8F=96=E6=96=B9?= =?UTF-8?q?=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 32 ++++++-- .../crawl/58/District58DetailCrawl.groovy | 73 +++++++++++++++++++ .../crawl/58/DistrictDetailCrawl.groovy | 46 ------------ .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 7 +- .../temp/shenjian/util/GaoDeUtils.groovy | 8 +- 5 files changed, 112 insertions(+), 54 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy delete mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index e95cbadff..10e63221e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -2,6 +2,11 @@ package com.fangjia.script.temp.shenjian import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject +import com.baidubce.http.ApiExplorerClient +import com.baidubce.http.AppSigner +import com.baidubce.http.HttpMethodName +import com.baidubce.model.ApiExplorerRequest +import com.baidubce.model.ApiExplorerResponse import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.SeaweedUtils import org.elasticsearch.geometry.utils.Geohash @@ -12,11 +17,28 @@ import org.elasticsearch.geometry.utils.Geohash class Temp { static void main(String[] args) { - String str = SeaweedUtils.encodeByGeoHash(106.569161, 29.498502) - println(str) - str = Geohash.stringEncode(106.569161, 29.498502, 10) - println(str) - // wm5zc4131rmf + String path = "https://goojia.api.bdymkt.com/estimate"; + ApiExplorerRequest request = new ApiExplorerRequest(HttpMethodName.GET, path); + request.setCredentials("fccf809a0f7748449154af926897aecb", "af1f0088a05c493ab20133ba470f87ef "); + + // 设置header参数 + request.addHeaderParameter("Content-Type", "application/json; charset=utf-8"); + + // 设置query参数 + request.addQueryParameter("address", "上海宝山呼玛三村"); + request.addQueryParameter("city", "上海"); + request.addQueryParameter("size", "100"); + + + ApiExplorerClient client = new ApiExplorerClient(new AppSigner()); + + try { + ApiExplorerResponse response = client.sendRequest(request); + // 返回结果格式为Json字符串 + System.out.println(response.getResult()); + } catch (Exception e) { + e.printStackTrace(); + } } // 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy new file mode 100644 index 000000000..d02c980c1 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/District58DetailCrawl.groovy @@ -0,0 +1,73 @@ +package com.fangjia.script.temp.shenjian.crawl + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j +import org.springframework.util.StopWatch + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 58小区详情爬取 + * + * @author shenjian* @since 2021/5/26 + */ +@Slf4j +class District58DetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(100000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("访问过于频繁") == -1 && content.indexOf("请输入验证码") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_58") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy deleted file mode 100644 index 095456eed..000000000 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/DistrictDetailCrawl.groovy +++ /dev/null @@ -1,46 +0,0 @@ -package com.fangjia.script.temp.shenjian.crawl - -import com.alibaba.fastjson.JSON -import com.alibaba.fastjson.JSONObject -import com.fangjia.script.temp.shenjian.util.MessageUtils -import com.fangjia.script.util.ABuYunUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection - -/** - * 58小区详情爬取 - * - * @author shenjian - * @since 2021/5/26 - */ -class DistrictDetailCrawl { - - static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) - - static void main(String[] args) { - districtSeedConn.find(new BasicDBObject("root", "58.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) - .batchSize(1000).limit(1000).each { obj -> - final def URL = "https://" + obj.city_short + ".58.com/xiaoqu/" + obj.district_id + "/" - def res = ABuYunUtil.getReponseContent2(URL, "utf-8") - if (res && res.code == 200) { - String content = res.content - - JSON json = new JSONObject() - json.put("project", "district_detail_58") - json.put("url", URL) - - JSON result = new JSONObject() - result.put("content", (String)content) - json.put("result", result) - - byte[] packedMsg = MessageUtils.pack(json) - CommonUtils.sendMessage("result2parse", packedMsg) - districtSeedConn.update(new BasicDBObject("_id", obj._id), - new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) - println("已抓取" + obj.district_id) - } - } - } -} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 016d7be75..4d7a51343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -29,11 +29,10 @@ class SeaweedAroundPoi { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" - static ExecutorService executorService = Executors.newFixedThreadPool(10) + static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { def types = listAllType() - //def types = "0601" DBObject query = new BasicDBObject() query.put("status", 1) conn136.find(query).batchSize(5000).each {objTemp -> @@ -46,6 +45,10 @@ class SeaweedAroundPoi { def lat = obj.lat List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") if (CollectionUtils.isNotEmpty(pois)) { + if (pois.get(0).id == "10044") { + executorService.shutdown() + throw new Error("当日调用已超限") + } String jsonStr = JSONObject.toJSONString(pois) HttpUtils.postForJson(URL, jsonStr) conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index e384c35fb..3f234cc53 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -8,6 +8,8 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import java.util.concurrent.ExecutorService + /** * 高德工具类 * @@ -138,7 +140,11 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10044") { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 - System.exit(1) + //System.exit(1) + AmapPoi amapPoi = new AmapPoi() + amapPoi.id = "10044" + result.add(amapPoi) + break; } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) From 6c19fafa521e307811403c8d00026199fd46f920 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 28 May 2021 09:08:31 +0800 Subject: [PATCH 062/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AE=89=E5=B1=85?= =?UTF-8?q?=E5=AE=A2=E5=B0=8F=E5=8C=BA=E8=AF=A6=E6=83=85=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E6=8A=93=E5=8F=96=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../anjuke/AnjukeDistrictDetailCrawl.groovy | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy new file mode 100644 index 000000000..c9caccb1a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/anjuke/AnjukeDistrictDetailCrawl.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.anjuke + +import com.alibaba.fastjson.JSON +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.MessageUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import groovy.util.logging.Slf4j + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 安居客小区详情抓取 + * + * @author shenjian + * @since 2021/5/28 + */ +@Slf4j +class AnjukeDistrictDetailCrawl { + + static DBCollection districtSeedConn = MongoUtils.getCollectionByName("seeddb", "district_seeds", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong() + CountDownLatch latch = new CountDownLatch(100000) + districtSeedConn.find(new BasicDBObject("root", "anjuke.com").append("status", new BasicDBObject('$eq', null)), new BasicDBObject("district_id", 1).append("city_short", 1).append("_id", 1)) + .batchSize(1000).limit(1000).each { objTemp -> + def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + final def URL = "https://" + obj.city_short + ".anjuke.com/community/view/" + obj.district_id + "#/" + def res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res && res.code == 200) { + String content = res.content + if (content.indexOf("请拖动滑块进行验证") == -1 && content.indexOf("访问验证-安居客") == -1) { + if (content.indexOf("您所访问的页面不存在") != -1) { + districtSeedConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "页面不存在"))) + } else { + JSON json = new JSONObject() + json.put("project", "district_detail_anjuke") + json.put("url", URL) + + JSON result = new JSONObject() + result.put("content", (String) content) + json.put("result", result) + + byte[] packedMsg = MessageUtils.pack(json) + CommonUtils.sendMessage("result2parse2dev", packedMsg) + districtSeedConn.update(new BasicDBObject("_id", obj._id), + new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已抓取"))) + log.info("已抓取 {}", obj.district_id) + } + } + } else if (res && res.code == 508) { + log.info("508 {}", obj.district_id) + } + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + } +} From 62a844316074db7f08e4ea039d4b165e48108bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 1 Jun 2021 16:01:48 +0800 Subject: [PATCH 063/212] =?UTF-8?q?=E7=BB=A9=E6=95=88=E4=BB=BB=E5=8A=A1?= =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=A1=B9=E7=9B=AE=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/dingding/TaskExport.groovy | 5 +---- .../script/temp/shenjian/util/GaoDeUtils.groovy | 10 +++++----- .../java/com/fangjia/script/statistics/ApiUtil.java | 4 +++- .../main/java/com/fangjia/script/statistics/Task.java | 11 +++++++++++ 4 files changed, 20 insertions(+), 10 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index a86e1a959..1cc8d65b5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -49,13 +49,10 @@ public class TaskExport { Thread.sleep(60 * 1000) taskDetail = ApiUtil.getTaskDetail(task.getId()) } - if ("穆良书" == task.getResponse().getName() && task.getId() == "e3f81b0381b44696b06fb6242a44902f") { - print("穆良书") - } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { TaskPersonDTO dto = new TaskPersonDTO() dto.name = task.response.name - dto.project = "" + dto.project = task.project.name dto.taskName = task.name dto.score = org.apache.commons.lang3.StringUtils.isBlank(taskDetail.flowConten) ? 0d : Double.valueOf(taskDetail.flowConten) dto.workTime = com.fangjia.script.common.DataHandleUtils.getDoubleRound(0, Double.valueOf(taskDetail.workTime)) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 3f234cc53..e16b6336a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -140,11 +140,11 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10044") { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 - //System.exit(1) - AmapPoi amapPoi = new AmapPoi() - amapPoi.id = "10044" - result.add(amapPoi) - break; + System.exit(1) +// AmapPoi amapPoi = new AmapPoi() +// amapPoi.id = "10044" +// result.add(amapPoi) +// break; } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 977bc2888..7c87580de 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -340,7 +340,9 @@ public class ApiUtil { JSONObject obj = JSONObject.parseObject(String.valueOf(result.get("data"))); detail = obj.toJavaObject(TaskDetail.class); } else if (code == 90001){ - throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:getTaskDetail"); + Thread.sleep(1000 * 60); + return getTaskDetail(taskId); + //throw new RuntimeException("姣忎釜浼佷笟璋冪敤鍗曚釜鎺ュ彛鐨勯鐜囦笉鍙秴杩�60娆�/鍒嗛挓锛屼細杩斿洖閿欒鐮�90001:getTaskDetail"); } else if (code == 90009) { throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:getTaskDetail"); } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java index 9a0c728c4..78fd1a825 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/Task.java @@ -32,6 +32,8 @@ public class Task { private FlowDTO flow; @JSONField(name = "createDate") private String createDate; + @JSONField(name = "project") + private ProjectDTO project; @NoArgsConstructor @Data @@ -61,4 +63,13 @@ public class Task { @JSONField(name = "userid") private String userid; } + + @NoArgsConstructor + @Data + public static class ProjectDTO { + @JSONField(name = "id") + private String id; + @JSONField(name = "proname") + private String name; + } } From ea3ae0cf612a1c1a52973f332c74114d82142790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 13:44:09 +0800 Subject: [PATCH 064/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E5=9F=8E=E5=B8=82=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/58/City58.groovy | 6 +- .../temp/shenjian/crawl/ke/CityKe.groovy | 57 +++++++++++++++++++ .../temp/shenjian/util/CityUtils.groovy | 19 +++++++ 3 files changed, 78 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy index 8b9abf395..5222957f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/58/City58.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.temp.shenjian.crawl import com.alibaba.fastjson.JSONObject +import com.fangjia.script.temp.shenjian.util.CityUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection @@ -252,16 +253,13 @@ class City58 { " }"; static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) - static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static Set citySet = CityUtils.listCitySet() static void main(String[] args) { JSONObject jsonObject = JSONObject.parseObject(cityList) Set provinceSet = jsonObject.keySet() List ret = new ArrayList<>() - List list = cityMapConn.distinct("fangjia_city") - Set citySet = new HashSet<>(list); - for (String province: provinceSet) { JSONObject provinceInfo = jsonObject[province] for (String city : provinceInfo.keySet()) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy new file mode 100644 index 000000000..e8ae7fc0c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CityKe.groovy @@ -0,0 +1,57 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + final def URL = "https://www.ke.com/city/" + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + Elements elements = JQueryUtils.select(".city_list_ul .city_list_li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText(".city_province .city_list_tit", element) + Elements cityElements = JQueryUtils.select(".CLICKDATA a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".ke.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "ke.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("贝壳找房城市数据写入成功") + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy new file mode 100644 index 000000000..2949d3803 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/CityUtils.groovy @@ -0,0 +1,19 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +/** + * @author shenjian + * @since 2021/6/2 + */ +class CityUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + + static listCitySet() { + List list = cityMapConn.distinct("fangjia_city") + Set citySet = new HashSet<>(list); + return citySet + } +} From 3395ae6248ebe35e8d04555949751ddde4163def Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 14:08:19 +0800 Subject: [PATCH 065/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E5=90=84=E5=9F=8E=E5=B8=82=E6=88=90=E4=BA=A4?= =?UTF-8?q?=E6=80=BB=E9=87=8F=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/ke/CountTradeKe.groovy | 75 +++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy new file mode 100644 index 000000000..064598c6b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -0,0 +1,75 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup +import org.jsoup.nodes.Document + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 贝壳各城市成交总量抓取 + * + * @author shenjian + * @since 2021/6/2 + */ +class CountTradeKe { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + crawlTradeCount() + //println(count()) + } + + static crawlTradeCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city_short", 1).append("city", 1)) + .batchSize(100).each { objTemp -> + final def obj = objTemp + executorService.execute(new Runnable() { + @Override + void run() { + def id = obj._id + def city_short = obj.city_short + final def URL = "https://" + city_short + ".ke.com/chengjiao/" + try { + String res = ABuYunUtil.getReponseContent2(URL, "utf-8") + if (res) { + Document doc = Jsoup.parse(res); + boolean existTrade = JQueryUtils.selectText(".resultDes .total", doc).contains("成交房源") + if (existTrade) { + String str = JQueryUtils.selectText(".resultDes .total span", doc) + int count = Integer.parseInt(str) + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", count))) + } else { + cityConn.update(new BasicDBObject("_id", id), new BasicDBObject('$set', new BasicDBObject("trade_count", 0))) + } + println("当前处理城市" + obj.city) + } + } catch(Exception ex) { + ex.printStackTrace() + } + } + }) + } + + executorService.shutdown() + } + + static count() { + int number = 0; + cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1)) + .batchSize(100).each { obj -> + if (obj.trade_count) { + number += obj.trade_count + } + } + return number + } +} From 62a419ea669e6bbe34c88fe2510c3244c40aea90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 14:36:29 +0800 Subject: [PATCH 066/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E8=B4=9D=E5=A3=B3?= =?UTF-8?q?=E6=89=BE=E6=88=BF=E6=8A=93=E5=8F=96=E5=BC=82=E5=B8=B8=E7=BB=9F?= =?UTF-8?q?=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/crawl/ke/CountTradeKe.groovy | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy index 064598c6b..d8b2c466b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/ke/CountTradeKe.groovy @@ -20,11 +20,14 @@ import java.util.concurrent.Executors class CountTradeKe { static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static DBCollection dealPriceConn = MongoUtils.getCollectionByName("fangjia", "deal_price", "mongo.db.fangjia.com") static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { - crawlTradeCount() + //crawlTradeCount() //println(count()) + //1896919 1760177 + findUnNormalCount() } static crawlTradeCount() { @@ -62,6 +65,28 @@ class CountTradeKe { executorService.shutdown() } + /** + * 找出异常抓取的城市 + */ + static findUnNormalCount() { + cityConn.find(new BasicDBObject("root", "ke.com"), + new BasicDBObject("_id", 1).append("city", 1).append("city_short", 1).append("trade_count", 1)) + .batchSize(100).each { obj -> + Long realTradeCount = obj.trade_count + if (realTradeCount > 0) { + Long crawledCount = dealPriceConn.count(new BasicDBObject("city", obj.city).append("source", "贝壳找房")) + if (realTradeCount > crawledCount) { + println(obj.city + " " + obj.city_short + "抓取异常:抓取为" + crawledCount + " 实际为" + realTradeCount + " 少抓" + (realTradeCount - crawledCount)) + } + } + } + } + + /** + * 统计实际抓取量 + * + * @return + */ static count() { int number = 0; cityConn.find(new BasicDBObject("root", "ke.com"), new BasicDBObject("trade_count", 1)) From 37d0ad878c6166d77300c92e4b5d9cf908f99c11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 2 Jun 2021 16:53:12 +0800 Subject: [PATCH 067/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=88=BF=E5=A4=A9?= =?UTF-8?q?=E4=B8=8B=E5=9F=8E=E5=B8=82=E7=A7=8D=E5=AD=90?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/crawl/fang/CityFang.groovy | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy new file mode 100644 index 000000000..88a3b6f38 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/crawl/fang/CityFang.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.crawl.ke + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.temp.shenjian.util.CityUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.lang3.StringUtils +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements + +/** + * 贝壳找房城市种子生成 + * + * @author shenjian + * @since 2021/6/2 + */ +class CityFang { + + static DBCollection cityConn = MongoUtils.getCollectionByName("seeddb", "city", "192.168.0.235", 27018) + static Set citySet = CityUtils.listCitySet() + + static void main(String[] args) { + // 房天下较为特殊,直接从url取乱码,需要改页面所有非UTF-8编码为UTF-8 + Document doc = Jsoup.parse(new File("F:\\数据\\爬虫\\fang_city.html"), "UTF-8"); + Elements elements = JQueryUtils.select(".outCont li", doc) + List ret = new ArrayList<>() + for (Element element : elements) { + String province = JQueryUtils.selectText("strong", element) + Elements cityElements = JQueryUtils.select("a", element) + for (Element cityElement: cityElements) { + String city = cityElement.text() + if ("直辖市".equals(province)) { + province = city + } + if (citySet.contains(city)) { + String url = cityElement.attr("href") + String cityShort = StringUtils.substringBetween(url,"//", ".esf.fang.com") + DBObject dbObject = new BasicDBObject() + dbObject.put("province", province) + dbObject.put("city", city) + dbObject.put("city_short", cityShort) + dbObject.put("c_date", new Date()) + dbObject.put("m_date", new Date()) + dbObject.put("root", "fang.com") + ret.add(dbObject) + } + } + } + cityConn.insert(ret) + println("房天下城市数据写入成功") + } +} From 995e92f4382f69edc811f0959497bc949ba17d8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 4 Jun 2021 10:53:46 +0800 Subject: [PATCH 068/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=87=8F=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/seaweed/CountSeaweed.groovy | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy new file mode 100644 index 000000000..ff7f7d322 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/CountSeaweed.groovy @@ -0,0 +1,36 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.fangjia.script.util.DbUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBObject + +/** + * @author shenjian + * @since 2021/6/4 + */ +class CountSeaweed { + static void main(String[] args) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("cat", "district"); + q.put("visible", 0); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + println(city + " " + DbUtils.seaweed.count(q)) + } + + } +} From 58f41e5154647764c94ef378d105bbeb317b5caa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 7 Jun 2021 10:14:21 +0800 Subject: [PATCH 069/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E9=87=8F=E7=BB=9F=E8=AE=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy | 4 ---- .../com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 4 ---- 2 files changed, 8 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 4d7a51343..a0227abe4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -45,10 +45,6 @@ class SeaweedAroundPoi { def lat = obj.lat List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") if (CollectionUtils.isNotEmpty(pois)) { - if (pois.get(0).id == "10044") { - executorService.shutdown() - throw new Error("当日调用已超限") - } String jsonStr = JSONObject.toJSONString(pois) HttpUtils.postForJson(URL, jsonStr) conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index e16b6336a..494e0cb22 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -141,10 +141,6 @@ class GaoDeUtils { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 System.exit(1) -// AmapPoi amapPoi = new AmapPoi() -// amapPoi.id = "10044" -// result.add(amapPoi) -// break; } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) From 2da2fb9e4216b4c2a5b8a2abd649cef80d0b1492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 7 Jun 2021 13:57:12 +0800 Subject: [PATCH 070/212] =?UTF-8?q?=E4=BC=98=E5=85=88=E5=A4=84=E7=90=86?= =?UTF-8?q?=E9=87=8D=E5=BA=86=E5=86=9C=E5=95=86=E8=A1=8C=E6=8B=9B=E6=A0=87?= =?UTF-8?q?=E5=9F=8E=E5=B8=82POI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 32 +++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index a0227abe4..addf2d9cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -33,7 +33,36 @@ class SeaweedAroundPoi { static void main(String[] args) { def types = listAllType() - DBObject query = new BasicDBObject() + dealSpecialSeaweedAround(types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { query.put("status", 1) conn136.find(query).batchSize(5000).each {objTemp -> final def obj = objTemp @@ -55,7 +84,6 @@ class SeaweedAroundPoi { } }) } - executorService.shutdown() } static getNewSeaweed() { From e4ac06de8c236df297935752f1d2a5e9fe6cb924 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 11 Jun 2021 15:27:00 +0800 Subject: [PATCH 071/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=85=B6=E4=BB=96?= =?UTF-8?q?=E7=9B=91=E6=8E=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/SeawaterMonitor.groovy | 58 ++++++++++++++++++- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 3 +- 2 files changed, 59 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy index 8627efc19..34df24342 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SeawaterMonitor.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.util.DingDingMessageUtil import org.apache.commons.lang.StringUtils +import org.apache.commons.lang3.time.DateFormatUtils /** * 抓取总量监控 @@ -37,7 +38,16 @@ class SeawaterMonitor extends BaseBatch { static final site = ["lianjia.com", "ke.com", "58.com", "zhuge.com", "fang.com"] static void main(String[] args) { - runBatch(new SeawaterMonitor(), args) + //runBatch(new SeawaterMonitor(), args) + //monitorDayCrawlReasonable() + def results = [] + results << ["日期", "网站", "出售", "出租"] + for (int i = 1; i < 90; i++) { + def startDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i).getTime() + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + countDayCrawlBySite(startDate, endDate, results) + } + ExcelUtils.write("f://数据//爬虫//分析.xlsx", results, "sheet1") } /** @@ -125,6 +135,52 @@ class SeawaterMonitor extends BaseBatch { return "" } + /** + * 统计近90天更新量超过40万的天数 + * @param args + */ + static def monitorDayCrawlReasonable() { + def map = [:] + for (int i = 1; i < 90; i++) { + def date = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i) + def startDate = date.getTime(); + def endDate = DateUtil.add(DateUtil.truncateDay(new Date()), -1 * i + 1).getTime() + def total = countDayCrawl(startDate, endDate) + if (total >= expectNumber) { + map.put(DateUtil.formatToYYYYMMDD(date), total) + } + } + println("近90天更新数量超过" + expectNumber + "的天数为" + map.size() + ",分别为:" ) + for (def key : map.keySet()) { + println(key + " : " + map.get(key)) + } + } + + /** + * 统计每个网站时间段范围内的更新量 + */ + static def countDayCrawlBySite(def startDate, def endDate, def results) { + def sites = ["lianjia.com", "ke.com", "fang.com", "58.com", "anjuke.com"] + def sql = "select count(*) as total, root, cat from seawater where m_date >= '${startDate}' and m_date < '${endDate}' group by root, cat" + def map = [:] + ElasticUtils.executeSqlQuery(sql,{data-> + map.put(data.root + "|" + data.cat, data.total) + }); + for (def site : sites) { + results << [DateFormatUtils.format(startDate, "yyyy-MM-dd"), site, map.get(site + "|sell"), map.get(site + "|rent")] + } + return results + }; + + static def countDayCrawl(def startDate, def endDate) { + def total = 0 + def sql = "select count(*) as total from seawater where m_date >= '${startDate}' and m_date < '${endDate}'" + ElasticUtils.executeSqlQuery(sql,{data-> + total = data.total + }) + return total + }; + @Override void process(Object args) { def lastWeekDate = DateUtil.add(DateUtil.truncateDay(new Date()), -7).getTime() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index addf2d9cf..7a10c9035 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -33,7 +33,8 @@ class SeaweedAroundPoi { static void main(String[] args) { def types = listAllType() - dealSpecialSeaweedAround(types) + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) executorService.shutdown() } From f51d9ed812d3bebfe95d0b7929e530219f243c85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 17:32:03 +0800 Subject: [PATCH 072/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=AB=98=E5=BE=B7key?= =?UTF-8?q?=E5=A4=9A=E7=BA=BF=E7=A8=8B=E4=BD=BF=E7=94=A8=E5=AE=8C=E5=81=9C?= =?UTF-8?q?=E6=AD=A2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 494e0cb22..5adde74e3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -8,7 +8,7 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import java.util.concurrent.ExecutorService +import java.util.concurrent.atomic.AtomicInteger /** * 高德工具类 @@ -19,7 +19,8 @@ import java.util.concurrent.ExecutorService class GaoDeUtils { // 新获取key - def static GAO_DE_AK = "4e9592208336433ee1b7aa4d1d434418" + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) def static codeNameMap = new HashMap(){{ put("150500", "地铁"); @@ -59,7 +60,7 @@ class GaoDeUtils { */ static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] - params.key = GAO_DE_AK + params.key = GAO_DE_AK[keyUsed] if (types) { params.types = types @@ -108,7 +109,7 @@ class GaoDeUtils { */ static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ def params = [:] - params.key = GAO_DE_AK + params.key = GAO_DE_AK[keyUsed.intValue()] if (types) { params.types = types @@ -140,7 +141,10 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10044") { //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") // 多线程情况下还是exit有效果,抛异常不会停止程序 - System.exit(1) + if (keyUsed.intValue() >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed.incrementAndGet() } else if (json && json.infocode && json.infocode == "10004") { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) From b7ae62d5821c90ef563d4e88e114243292cd6b68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 17:59:35 +0800 Subject: [PATCH 073/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9POI?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 7a10c9035..476a42a3a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -1,20 +1,16 @@ package com.fangjia.script.temp.shenjian.poi - import com.alibaba.fastjson.JSONObject -import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.temp.shenjian.util.HttpUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.MongoUtils -import com.fangjia.script.wenyan.auto.KeywordSearchUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.commons.collections.CollectionUtils -import org.bson.types.ObjectId import java.util.concurrent.ExecutorService import java.util.concurrent.Executors @@ -154,10 +150,24 @@ class SeaweedAroundPoi { static def listAllType() { StringBuilder ret = new StringBuilder() - for (def val : KeywordSearchUtils.KEYWORDS_MAP.values()) { + for (def val : KEYWORDS_MAP.values()) { ret.append(val).append("|") } String result = ret.toString() return result.substring(0, result.length() - 1) } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; } From 6d6c7cab29c33628c611768f5c3ffc9bfa96a485 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 18:08:24 +0800 Subject: [PATCH 074/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=91=A8=E8=BE=B9POI?= =?UTF-8?q?=E6=90=9C=E7=B4=A2=E5=AE=9A=E6=97=B6=E4=BB=BB=E5=8A=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 185 ++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 181 +++++++++++++++ .../temp/shenjian/util/HttpUtils.groovy | 29 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 208 ++++++++++++++++++ .../script/statistics/poi/AmapPoi.java | 71 ++++++ 6 files changed, 680 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..75ca2b1ff 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -482,7 +482,12 @@ j2v8_win32_x86_64 4.6.0 - + + org.projectlombok + lombok + 1.18.20 + provided + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..00ee277ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..5adde74e3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,181 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + //throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + // 多线程情况下还是exit有效果,抛异常不会停止程序 + if (keyUsed.intValue() >= GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed.incrementAndGet() + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..9fa4739f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,208 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2000 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From 7042809ecd9bf1fa3af87232fa71c16f124fd914 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 18 Jun 2021 18:08:24 +0800 Subject: [PATCH 075/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=A4=9A=E7=BA=BF?= =?UTF-8?q?=E7=A8=8B=E5=A4=9Akey=E5=81=9C=E6=AD=A2bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 7 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 185 ++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 186 ++++++++++++++++ .../temp/shenjian/util/HttpUtils.groovy | 29 +++ .../temp/shenjian/util/SeaweedUtils.groovy | 208 ++++++++++++++++++ .../script/statistics/poi/AmapPoi.java | 71 ++++++ 6 files changed, 685 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index ee26c1830..75ca2b1ff 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -482,7 +482,12 @@ j2v8_win32_x86_64 4.6.0 - + + org.projectlombok + lombok + 1.18.20 + provided + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy new file mode 100644 index 000000000..00ee277ea --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边POI数据获取 + * + * @author shenjian + * @since 2021/5/24 + */ +@BatchOwner(Watcher.ShenJian) +class SeaweedAroundPoi extends BaseBatch { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + /** + * 有限处理招标城市 + */ + static dealSpecialSeaweedAround(def types) { + def cities = ["重庆", "达州", "曲靖", "昆明", "大理", "迪庆州", "福安", "平潭", "三明", "石狮", "柳州", "苏州"] + for (def city : cities) { + DBObject q = new BasicDBObject(); + q.put("city", city); + if (city == "福安") { + city = "宁德" + q.put("city", city) + q.put("region", "福安") + } else if (city == "平潭") { + city = "福州" + q.put("city", city) + q.put("region", "平潭") + } else if (city == "石狮") { + city = "泉州" + q.put("city", city) + q.put("region", "石狮") + } + dealPoi(q, types) + } + } + + static dealPoi(DBObject query, def types) { + query.put("status", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("flag", "已获取周边POI"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "周边POI为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } + + static getNewSeaweed() { + DBObject query = new BasicDBObject() + query.put("cat", "district") + query.put("visible", 0) + //query.put("_id", new ObjectId("50c96f82e4b0e15299cb018a")) + + DBObject filter = new BasicDBObject() + filter.put("_id", 1) + filter.put("city", 1) + filter.put("region", 1) + filter.put("name", 1) + filter.put("lng", 1) + filter.put("lat", 1) + + List list = [] + DbUtils.seaweed.find(query, filter).batchSize(10000).each {obj -> + if (conn136.count(new BasicDBObject("_id", obj._id)) > 0) { + // 已存在小区,无需同步 + } else { + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + list << newObj + if (list.size() == 5000) { + conn136.insert(list) + list.clear() + } + } + } + conn136.insert(list) + list.clear() + } + + static def getByIdAndInsert(String id) { + BasicDBObject obj = DbUtils.seaweed.findById(id); + DBObject newObj = new BasicDBObject() + newObj.put("_id", obj._id) + newObj.put("city", obj.city) + newObj.put("region", obj.region) + newObj.put("name", obj.name) + try { + newObj.put("lng", SeaweedUtils.formatLatLng(obj.lng)) + newObj.put("lat", SeaweedUtils.formatLatLng(obj.lat)) + } catch (Exception ex) { + print("经纬度为空:" + obj._id + " " + obj) + ex.printStackTrace() + } + newObj.put("status", 1) + newObj.put("flag", "未获取周边POI") + + conn136.insert(newObj) + } + + static def listAllType() { + StringBuilder ret = new StringBuilder() + for (def val : KEYWORDS_MAP.values()) { + ret.append(val).append("|") + } + String result = ret.toString() + return result.substring(0, result.length() - 1) + } + + public static final Map KEYWORDS_MAP = new HashMap(){{ + put("地铁", "150500"); + put("公交", "1507"); + put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); + put("写字楼", "120201"); + put("学校","141200|141201|141202|141203|141204"); + put("银行","1601|1603"); + put("大型超市","0604"); + put("商场","0601"); + put("餐饮","0501|0502|0503|0504"); + put("旅游景点","110000"); + put("图书馆","140500"); + }}; + + @Override + void process(Object args) { + def types = listAllType() + //dealSpecialSeaweedAround(types) + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy new file mode 100644 index 000000000..74505e6b5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -0,0 +1,186 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.fastjson.JSONArray +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.DataUtil +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.HttpRequestUtil +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils + +import java.util.concurrent.atomic.AtomicInteger + +/** + * 高德工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class GaoDeUtils { + + // 新获取key + def static GAO_DE_AK = ["4e9592208336433ee1b7aa4d1d434418", "35925efd7333f1df52145e51b8e4e56a"] + def static AtomicInteger keyUsed = new AtomicInteger(0) + + def static codeNameMap = new HashMap(){{ + put("150500", "地铁"); + put("1507", "公交"); + put("090100", "医院") + put("090101", "医院") + put("090200", "医院") + put("090202", "医院") + put("090202", "医院") + put("090204", "医院") + put("090205", "医院") + put("090206", "医院") + put("090206", "医院") + put("090208", "医院") + put("090209", "医院") + put("090210", "医院") + put("090211", "医院") + put("141200", "学校") + put("141201", "学校") + put("141202", "学校") + put("141203", "学校") + put("141204", "学校") + put("1601", "银行") + put("1603", "银行") + put("0601", "商场") + }}; + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types "150500|1507|090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211|1601|1603|0601"; @see com.fangjia.script.wenyan.auto.KeywordSearchUtils.KEYWORDS_MAP.get(kw) + * @return + */ + static List searchNearByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + result.addAll(DataUtil.toMongoDbObj(json.pois)) + } else if (json && json.infocode && json.infocode == 10003) { + throw new RuntimeException("开发者的日访问量超限,被系统自动封停,第二天0:00会自动解封") + } else if (json && json.infocode && json.infocode == 10004) { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 查找周边配套 + * @param city + * @param kw + * @param lng + * @param lat + * @param radius + * @param types + * @return + */ + static List searchNearDtoByPoi(String city, String types, Double lng, Double lat, Integer radius, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + + if (types) { + params.types = types + } else { + params.keywords = keywords + } + params.city = city + params.radius = String.valueOf(radius) + params.location = "${lng},${lat}".toString() + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else { + break + } + } + return result + } + + /** + * 获取经纬度之间距离 + */ + static def getDistance(double Lat, double Lng, double Lat1, double Lng1) { + return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) + } + + /** + * 获取地址经纬度 + */ + static def parserByGaode(def city, def address) { + JSONObject json = GaoDeApiUtils.geo(city, address) + if (json && json.status == "1") { + def ret = json.geocodes + if (ret) { + def str = ret.get(0).location + def level = ret.get(0).level + def arrs = str.split(",") + return ["lng": arrs[0], "lat": arrs[1]] + } + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy new file mode 100644 index 000000000..ff11d6991 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/HttpUtils.groovy @@ -0,0 +1,29 @@ +package com.fangjia.script.temp.shenjian.util + +import org.apache.commons.httpclient.HttpClient +import org.apache.commons.httpclient.methods.PostMethod +import org.apache.commons.httpclient.methods.StringRequestEntity + +/** + * + * @author shenjian + * @since 2021/5/25 + */ +class HttpUtils { + + static boolean postForJson(String url, String jsonString) { + HttpClient httpClient = new HttpClient() + StringRequestEntity requestEntity = new StringRequestEntity( + jsonString, + "application/json", + "UTF-8"); + + PostMethod postMethod = new PostMethod(url) + postMethod.setRequestEntity(requestEntity); + int statusCode = httpClient.executeMethod(postMethod); + if (statusCode == 200) { + return true + } + return false + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy new file mode 100644 index 000000000..9fa4739f8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -0,0 +1,208 @@ +package com.fangjia.script.temp.shenjian.util + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.datastore.FieldValidatorUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.elasticsearch.geometry.utils.Geohash + +import java.util.concurrent.ConcurrentHashMap + +/** + * 小区工具类 + * + * @author shenjian + * @since 2021/5/17 + */ +class SeaweedUtils { + + static DBCollection cityMapConn = MongoUtils.getCollectionByName("business", "fangjia_official_area_map", "192.168.0.43", 28017) + static def map = new ConcurrentHashMap() + + static getRegionByCode(def district_code) { + def region; + if (map.get("district_code") != null) { + region = map.get("district_code") + } else { + cityMapConn.find(new BasicDBObject("district_code", district_code)).each { obj -> + region = obj.fangjia_district + map.put("district_code", region) + } + } + return region + } + + /** + * 标准化输出绿化率 + */ + static formatVirescencePercent(def virescence_percent) { + // 绿化率 + if (virescence_percent && FieldValidatorUtils.checkValue(virescence_percent)) { + double val = Double.valueOf(virescence_percent) + if (val < 1) { + return val * 100 + } + if (val == 1) { + return val * 10 + } + if (val >= 70) { + return 65 + } + return val + } + return "" + } + + /** + * 标准化容积率输出 + */ + static formatCubagePercent(def cubage_percent) { + if (cubage_percent && FieldValidatorUtils.checkValue(cubage_percent)) { + double val = Double.valueOf(cubage_percent) + if (val >= 7 && val < 8) { + return 6.2 + } + if (val >= 8 && val < 9) { + return 6.3 + } + if (val >= 9 && val < 10) { + return 6.5 + } + if (val >= 10) { + return 6.7 + } + return val + } + return "" + } + + /** + * 标准化车位输出 + */ + static formatCarSiteCount(def car_site_count) { + if (car_site_count && FieldValidatorUtils.checkValue(car_site_count)) { + long val = Long.valueOf(car_site_count) + if (val > 2000 || val < 100) { + return "" + } + return val; + } + return "" + } + + /** + * 标准化建筑面积输出 + */ + static formatBuildArea(def build_area) { + if (build_area && FieldValidatorUtils.checkValue(build_area)) { + double val = build_area + if (val > 1000) { + return val + } + } + return "" + } + + /** + * 标准化总占地面积输出 + */ + static formatAllArea(def all_area) { + if (all_area) { + Double formattedVal = 0.0; + if (all_area.toString().indexOf("公顷") != -1) { + def val = all_area.toString() + formattedVal = (Double.valueOf(val.substring(0, val.indexOf("公顷"))) * 10000) + } else if(FieldValidatorUtils.checkValue(all_area)) { + formattedVal = Double.valueOf(all_area) + } + if (formattedVal > 100000 || formattedVal == 0.0) { + return "" + } + return formattedVal + } + } + + /** + * 标准化户数输出 + */ + static formatHouseholdCount(def household_count) { + if (household_count && FieldValidatorUtils.checkValue(household_count) ) { + if (Long.valueOf(household_count) < 50) { + return "" // 户数 + } + return household_count + } + } + + /** + * 计算车位比 + */ + static calcCarAndHouseholdRadio(def car_site_count, def household_count) { + if (SeaweedUtils.formatCarSiteCount(car_site_count) && SeaweedUtils.formatHouseholdCount(household_count)) { + long formattedCarSiteCount = SeaweedUtils.formatCarSiteCount(car_site_count) + long formattedHouseholdCount = SeaweedUtils.formatHouseholdCount(household_count) + double val = DataHandleUtils.getDoubleRound(1, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(2, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 0.0) { + val = DataHandleUtils.getDoubleRound(3, (Double.valueOf(formattedCarSiteCount) / Double.valueOf(formattedHouseholdCount))) + } + if (val == 1.0) { + return "1:1" + } + return "1:" + val + } + return "" + } + + /** + * 标准化别名输出 + */ + static formatAlias(def alias) { + if (alias) { + // 小区可能存在多个别名 + def temp = [] + int len = 0 + for (def a : alias) { + len += a.toString().length() + if (len > 20) { + return temp.join(",") + } + temp.add(a) + } + return temp.join(",") + } + return "" + } + + /** + * 格式化经纬度 + */ + static def formatLatLng(def latOrLng) { + // 107.109878,29.192381 + return com.fangjia.script.common.DataHandleUtils.getDoubleRound(6, latOrLng / 10000000000d) + } + + /** + * 小区经纬度geohash编码 + * @param args + */ + static def encodeByGeoHash(double lon, double lat, int level=7) { + return Geohash.stringEncode(lon, lat, level) + } + + /** + * 小区经纬度geohash解码 + * @param args + */ + static def decodeByGeoHash(String geohash) { + return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + } + + static void main(String[] args) { + def alias = ["金杏一村","金杏一村","金杏一村","金杏一村","金杏一村", "金杏二村"] + println(formatAlias(alias)) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java new file mode 100644 index 000000000..4a731fe11 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/poi/AmapPoi.java @@ -0,0 +1,71 @@ +package com.fangjia.script.statistics.poi; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/5/25 + */ +@lombok.NoArgsConstructor +@lombok.Data +public class AmapPoi { + + @com.alibaba.fastjson.annotation.JSONField(name = "id") + private String id; + @com.alibaba.fastjson.annotation.JSONField(name = "parent") + private List parent; + @com.alibaba.fastjson.annotation.JSONField(name = "childtype") + private List childtype; + @com.alibaba.fastjson.annotation.JSONField(name = "name") + private String name; + @com.alibaba.fastjson.annotation.JSONField(name = "type") + private String type; + @com.alibaba.fastjson.annotation.JSONField(name = "typecode") + private String typecode; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_type") + private List bizType; + @com.alibaba.fastjson.annotation.JSONField(name = "address") + private String address; + @com.alibaba.fastjson.annotation.JSONField(name = "location") + private String location; + @com.alibaba.fastjson.annotation.JSONField(name = "tel") + private String tel; + @com.alibaba.fastjson.annotation.JSONField(name = "pname") + private String pname; + @com.alibaba.fastjson.annotation.JSONField(name = "cityname") + private String cityname; + @com.alibaba.fastjson.annotation.JSONField(name = "adname") + private String adname; + @com.alibaba.fastjson.annotation.JSONField(name = "importance") + private List importance; + @com.alibaba.fastjson.annotation.JSONField(name = "shopid") + private List shopid; + @com.alibaba.fastjson.annotation.JSONField(name = "shopinfo") + private String shopinfo; + @com.alibaba.fastjson.annotation.JSONField(name = "poiweight") + private List poiweight; + @com.alibaba.fastjson.annotation.JSONField(name = "distance") + private String distance; + @com.alibaba.fastjson.annotation.JSONField(name = "biz_ext") + private BizExtDTO bizExt; + @com.alibaba.fastjson.annotation.JSONField(name = "photos") + private List photos; + + @lombok.NoArgsConstructor + @lombok.Data + public static class BizExtDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "rating") + private List rating; + @com.alibaba.fastjson.annotation.JSONField(name = "cost") + private String cost; + } + + @lombok.NoArgsConstructor + @lombok.Data + public static class PhotosDTO { + @com.alibaba.fastjson.annotation.JSONField(name = "title") + private List title; + @com.alibaba.fastjson.annotation.JSONField(name = "url") + private String url; + } +} From 975dc9094752124015d68757b16e2474654471e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 22 Jun 2021 13:45:54 +0800 Subject: [PATCH 076/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=8C=96=E6=89=B9=E9=87=8F=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/address/StandardTest.groovy | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy new file mode 100644 index 000000000..3b3cca4f5 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy @@ -0,0 +1,60 @@ +package com.fangjia.script.temp.shenjian.address + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.ExcelUtils + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 地址标准化测试 + * + * @author shenjian + * @since 2021/6/22 + */ +class StandardTest { + + static ExecutorService executorService = Executors.newFixedThreadPool(10) + + static void main(String[] args) { + AtomicLong count = new AtomicLong(0) + List> data = ExcelUtils.read("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据.xlsx") + CountDownLatch latch = new CountDownLatch(data.size()) + for (List rowT : data) { + final row = rowT; + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = row[0] + def city = row[2] + HttpRequest request = new HttpRequest() + + String result = request.get("http://192.168.20.86:8082/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") + JSONObject object = JSONObject.parse(result); + if (object.get("status") == 0) { + row[7] = (((JSONObject) object.get("result")).get("formatAddress")) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + count.incrementAndGet() + if (count.intValue() % 1000 == 0) { + Thread.sleep(5000) + println(count) + } + } + } + }) + } + latch.await() + def headers = ["原始地址", "XH", "城市", "区域","小区名", "标注", "备注", "标准化地址"] + data.add(0, headers) + ExcelUtils.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx", data, "测试数据") + executorService.shutdown() + } +} From f4bd346bb43a0b733fb8519df3a6f272b75afabf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 22 Jun 2021 14:21:37 +0800 Subject: [PATCH 077/212] =?UTF-8?q?=E4=BB=8E=E6=95=B0=E6=8D=AE=E5=BA=93?= =?UTF-8?q?=E8=AF=BB=E5=8F=96=E5=BE=85=E6=A0=87=E5=87=86=E5=8C=96=E8=BE=93?= =?UTF-8?q?=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/ExportStandardResult.groovy | 63 +++++++++++++++++++ .../temp/shenjian/address/StandardTest.groovy | 25 ++++---- 2 files changed, 77 insertions(+), 11 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy new file mode 100644 index 000000000..bc2365983 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy @@ -0,0 +1,63 @@ +package com.fangjia.script.temp.shenjian.address + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 导出标准化输出结果 + * + * @author shenjian + * @since 2021/6/22 + */ +class ExportStandardResult { + static ExecutorService executorService = Executors.newFixedThreadPool(30) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) + + static void main(String[] args) { + List> data = new ArrayList<>() + def map = new ConcurrentHashMap() + CountDownLatch latch = new CountDownLatch(conn136.count().intValue()) + conn136.find(new BasicDBObject(), new BasicDBObject("address", 1).append("city", 1) + .append("format_address", 1).append("order", 1)).batchSize(10000).each { + objT -> + final obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def ret = [] + ret[0] = obj.city + ret[1] = obj.address + ret[2] = obj.format_address + map.put(obj.order, ret) + } catch (Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + for (int i = 0; i < map.size(); i++) { + data.add(map.get(i)) + } + def headArr = ["城市", "地址", "标准化输出"] + List head = new ArrayList<>() + for (String name: headArr) { + List temp = new ArrayList<>() + temp.add(name) + head.add(temp) + } + EasyExcel.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx").head(head).sheet("测试数据").doWrite(data); + + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy index 3b3cca4f5..b42c0b70d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy @@ -3,6 +3,9 @@ package com.fangjia.script.temp.shenjian.address import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.HttpRequest import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -18,25 +21,28 @@ import java.util.concurrent.atomic.AtomicLong class StandardTest { static ExecutorService executorService = Executors.newFixedThreadPool(10) + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) static void main(String[] args) { AtomicLong count = new AtomicLong(0) - List> data = ExcelUtils.read("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据.xlsx") - CountDownLatch latch = new CountDownLatch(data.size()) - for (List rowT : data) { - final row = rowT; + CountDownLatch latch = new CountDownLatch(conn136.count(new BasicDBObject('status', 1)).intValue()) + conn136.find(new BasicDBObject('status', 1)).batchSize(3000).each {objT -> + final def obj = objT executorService.submit(new Runnable() { @Override void run() { try { - def address = row[0] - def city = row[2] + def address = obj.address + def city = obj.city == null ? "" : obj.city HttpRequest request = new HttpRequest() String result = request.get("http://192.168.20.86:8082/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") JSONObject object = JSONObject.parse(result); if (object.get("status") == 0) { - row[7] = (((JSONObject) object.get("result")).get("formatAddress")) + def format_address = (((JSONObject) object.get("result")).get("formatAddress")) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("format_address", format_address).append("flag", "已标准化输出"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "标准化输出异常"))) } } catch(Exception ex) { ex.printStackTrace() @@ -50,11 +56,8 @@ class StandardTest { } } }) - } + }; latch.await() - def headers = ["原始地址", "XH", "城市", "区域","小区名", "标注", "备注", "标准化地址"] - data.add(0, headers) - ExcelUtils.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx", data, "测试数据") executorService.shutdown() } } From 714751cc9adf45156d5ddac155680bd11189c087 Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Tue, 22 Jun 2021 18:39:01 +0800 Subject: [PATCH 078/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E6=89=80=E8=AF=B7=E6=B1=82=E8=B5=84=E6=BA=90?= =?UTF-8?q?=E4=B8=8D=E5=8F=AF=E7=94=A8=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 74505e6b5..002e6aa14 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -154,6 +154,9 @@ class GaoDeUtils { // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 Thread.sleep(60000) break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) } else { break } From a8270b4f6825a110b33309fa3eb3a2d3f524b157 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 28 Jun 2021 16:00:03 +0800 Subject: [PATCH 079/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E6=A0=87=E5=87=86?= =?UTF-8?q?=E5=8C=96=E5=AF=BC=E5=87=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/ExportStandardResult.groovy | 31 +++++++++--- .../temp/shenjian/address/StandardTest.groovy | 50 ++++++++++++++++--- 2 files changed, 66 insertions(+), 15 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy index bc2365983..7b400357a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/ExportStandardResult.groovy @@ -4,6 +4,7 @@ import com.alibaba.excel.EasyExcel import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch @@ -23,9 +24,10 @@ class ExportStandardResult { static void main(String[] args) { List> data = new ArrayList<>() def map = new ConcurrentHashMap() - CountDownLatch latch = new CountDownLatch(conn136.count().intValue()) - conn136.find(new BasicDBObject(), new BasicDBObject("address", 1).append("city", 1) - .append("format_address", 1).append("order", 1)).batchSize(10000).each { + // 查询条件 + DBObject query = new BasicDBObject('status', 0).append('flag', '浙商疑问地址复测') + CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue()) + conn136.find(query).batchSize(10000).each { objT -> final obj = objT executorService.submit(new Runnable() { @@ -33,9 +35,22 @@ class ExportStandardResult { void run() { try { def ret = [] - ret[0] = obj.city - ret[1] = obj.address - ret[2] = obj.format_address + ret << obj.format_province + ret << obj.format_city + ret << obj.format_region + ret << obj.format_dev + ret << obj.format_township + ret << obj.format_community + ret << obj.format_group + ret << obj.format_street + ret << obj.format_streetNumber + ret << obj.format_name + ret << obj.format_buildingNumber + ret << obj.format_unitNumber + ret << obj.format_roomNumber + ret << obj.format_floor + ret << obj.format_address + map.put(obj.order, ret) } catch (Exception ex) { ex.printStackTrace() @@ -49,14 +64,14 @@ class ExportStandardResult { for (int i = 0; i < map.size(); i++) { data.add(map.get(i)) } - def headArr = ["城市", "地址", "标准化输出"] + def headArr = ["省","市","区","开发区","街道乡镇","居委会/社区", "组","路","路号","名称","楼栋", "单元","室号","楼层","优化后标准化地址"] List head = new ArrayList<>() for (String name: headArr) { List temp = new ArrayList<>() temp.add(name) head.add(temp) } - EasyExcel.write("F:\\工作\\基础建设\\标准化\\标准化融合后测试数据(标准化后).xlsx").head(head).sheet("测试数据").doWrite(data); + EasyExcel.write("F:\\工作\\基础建设\\标准化\\浙商疑问地址复测.xlsx").head(head).sheet("测试数据").doWrite(data); executorService.shutdown() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy index b42c0b70d..8611995d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/StandardTest.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -22,11 +23,13 @@ class StandardTest { static ExecutorService executorService = Executors.newFixedThreadPool(10) static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "address_standard", "192.168.0.136", 27017) - + static final String IP = "192.168.0.19:39080" static void main(String[] args) { AtomicLong count = new AtomicLong(0) - CountDownLatch latch = new CountDownLatch(conn136.count(new BasicDBObject('status', 1)).intValue()) - conn136.find(new BasicDBObject('status', 1)).batchSize(3000).each {objT -> + // 查询条件 + DBObject query = new BasicDBObject('status', 1).append('flag', '浙商疑问地址复测') + CountDownLatch latch = new CountDownLatch(conn136.count(query).intValue()) + conn136.find(query).batchSize(3000).each {objT -> final def obj = objT executorService.submit(new Runnable() { @Override @@ -36,11 +39,44 @@ class StandardTest { def city = obj.city == null ? "" : obj.city HttpRequest request = new HttpRequest() - String result = request.get("http://192.168.20.86:8082/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") + String result = request.get("http://" + IP + "/FangJiaAddressService/structure?city=" + URLEncoder.encode(city, "utf-8") + "&address=" + URLEncoder.encode(address, "utf-8"), "utf-8") JSONObject object = JSONObject.parse(result); if (object.get("status") == 0) { - def format_address = (((JSONObject) object.get("result")).get("formatAddress")) - conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0).append("format_address", format_address).append("flag", "已标准化输出"))) + JSONObject jsonResult = (JSONObject) object.get("result"); + def format_province = jsonResult.get("province") + def format_city = jsonResult.get("city") + def format_region = jsonResult.get("district") + def format_dev = jsonResult.get("dev") + def format_township = jsonResult.get("township") + def format_community = jsonResult.get("community") + def format_group = jsonResult.get("group") + def format_street = jsonResult.get("street") + def format_subStreet = jsonResult.get("subStreet") + def format_streetNumber = jsonResult.get("streetNumber") + def format_name = jsonResult.get("name") + def format_buildingNumber = jsonResult.get("buildingNumber") + def format_unitNumber = jsonResult.get("unitNumber") + def format_floor = jsonResult.get("floor") + def format_roomNumber = jsonResult.get("roomNumber") + def format_address = jsonResult.get("formatAddress") + + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 0) + .append("format_province", format_province) + .append("format_city", format_city) + .append("format_region", format_region) + .append("format_dev", format_dev) + .append("format_township", format_township) + .append("format_community", format_community) + .append("format_group", format_group) + .append("format_street", format_street) + .append("format_subStreet", format_subStreet) + .append("format_streetNumber", format_streetNumber) + .append("format_name", format_name) + .append("format_buildingNumber", format_buildingNumber) + .append("format_unitNumber", format_unitNumber) + .append("format_floor", format_floor) + .append("format_roomNumber", format_roomNumber) + .append("format_address", format_address))) } else { conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status", 2).append("flag", "标准化输出异常"))) } @@ -57,7 +93,7 @@ class StandardTest { } }) }; - latch.await() + latch.await(); executorService.shutdown() } } From e788d211f1e317a47cdd918ae3bfacaacee13c4a Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 1 Jul 2021 18:08:11 +0800 Subject: [PATCH 080/212] =?UTF-8?q?=E7=BB=A9=E6=95=88=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E4=BD=8D=E7=BD=AE=E5=85=BC=E5=AE=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy | 2 +- .../fangjia/script/temp/shenjian/dingding/ProjectExport.groovy | 2 +- .../fangjia/script/temp/shenjian/dingding/ScoreExport.groovy | 2 +- .../com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 00e7644b8..9603351a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,7 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-04-01", "yyyy-MM-dd") ; + Date now = DateUtils.parseFromTightByPattern("2021-06-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy index efd9a2e3b..b2535679e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ProjectExport.groovy @@ -93,7 +93,7 @@ class ProjectExport { mergeData.add(mergeRowCel) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "项目统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy index 511c72846..6a10fff7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy @@ -140,7 +140,7 @@ public class ScoreExport { allTotal[2] = "人员绩效总计" sortedResult << allTotal - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", sortedResult, "绩效统计", mergeData); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy index 1cc8d65b5..d7958a211 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/TaskExport.groovy @@ -117,7 +117,7 @@ public class TaskExport { mergeData.add(data) } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData); + com.fangjia.script.util.ExcelUtils.write("D:\\数据\\绩效\\统计.xlsx", result, "任务统计", mergeData); } } From 37c2ee5c89681e975cf034e790749e33b9ba110d Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 1 Jul 2021 18:19:35 +0800 Subject: [PATCH 081/212] =?UTF-8?q?=E6=96=B0=E5=A2=9Eexcel=E8=AF=BB?= =?UTF-8?q?=E5=8F=96=E5=B7=A5=E5=85=B7=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/util/ExcelUtils.groovy | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy new file mode 100644 index 000000000..229499b80 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.excel.EasyExcelFactory +import com.alibaba.excel.ExcelReader +import com.alibaba.excel.context.AnalysisContext +import com.alibaba.excel.event.AnalysisEventListener + +/** + * excel工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class ExcelUtils { + + static List> read(String filePath) { + SimpleExcelListener listener = new SimpleExcelListener(filePath); + ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); + excelReader.readAll() + List> dataList = listener.getDataList(); + excelReader.finish() + return dataList; + } + + class SimpleExcelListener extends AnalysisEventListener { + + private List> dataList = new ArrayList<>() + + @Override + void invoke(Object o, AnalysisContext analysisContext) { + Map map = o; + dataList.add(new ArrayList(map.values())) + } + + @Override + void doAfterAllAnalysed(AnalysisContext analysisContext) { + + } + + List> getDataList() { + return dataList + } + + void setDataList(List> dataList) { + this.dataList = dataList + } + } +} From 6b58f0a85629eeed95a9d400d9c4cfe18d7c5690 Mon Sep 17 00:00:00 2001 From: sjshenjian Date: Thu, 1 Jul 2021 18:19:35 +0800 Subject: [PATCH 082/212] =?UTF-8?q?=E6=96=B0=E5=A2=9EPOI=E6=8A=93=E5=8F=96?= =?UTF-8?q?=E6=83=85=E5=86=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/dingding/ScoreExport.groovy | 3 - .../temp/shenjian/poi/StatisticsPoi.groovy | 59 +++++++++++++++++++ .../temp/shenjian/util/ExcelUtils.groovy | 48 +++++++++++++++ .../temp/shenjian/util/MyStringUtils.groovy | 40 +++++++++++++ 4 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy index 6a10fff7c..2218e2ec6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/ScoreExport.groovy @@ -69,9 +69,6 @@ public class ScoreExport { } if (taskDetail.getRealityEndDate() != null && taskDetail.getRealityEndDate() > monthStartTime && taskDetail.getRealityEndDate() < endTime) { boolean find = false - if (taskDetail.getResponse().getName() == '穆良书' && taskDetail.getName() == "模型级联算法方案设计") { - println('穆良书') - } for (int i = 4; i < sortedResult[0].size(); i++) { // 找到该项目下该负责人,则加工时 if (sortedResult[0][i] == taskDetail.getResponse().getName()) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy new file mode 100644 index 000000000..cc0cb39dd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/StatisticsPoi.groovy @@ -0,0 +1,59 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.MyStringUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +/** + * @author shenjian* @since 2021/7/7 + */ +class StatisticsPoi { + static final cities = ["北京","上海","广州","深圳","重庆","郑州","长沙","西安","成都","东莞","佛山","杭州","合肥","南京", + "青岛","沈阳","苏州","天津","武汉","珠海","中山","长春","扬州","烟台","徐州","无锡","温州","太原", + "台州","石家庄","绍兴","厦门","泉州","宁波","南通","南宁","南昌","兰州","常州","大连","福州","贵阳", + "哈尔滨","海口","惠州","济南","嘉兴","金华","昆明","遵义","淄博","株洲","镇江","肇庆","漳州","湛江", + "岳阳","银川","宜昌","盐城","芜湖","乌鲁木齐","唐山","泰州","汕头","三亚","清远","秦皇岛","马鞍山","洛阳", + "柳州","连云港","廊坊","九江","吉林","淮安","湖州","呼和浩特","沧州","保定","安庆","张家口","西宁","日照", + "拉萨","锦州","大同","常德","北海","文昌","陵水","保亭"] + + + static final types = ['地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆'] + + static void main(String[] args) { + def result = [] + + result << ['城市', '数据量', '地铁', '公交', '医院', '写字楼', '学校', '银行', '大型超市', '商场', '餐饮', '旅游景点', '图书馆'] + for (def city in cities) { + def ret = [] + def total + String sql = "select count(*) total from amap_poi where cityname = '" + city + "' and status <> 1 " + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.total + }) + ret << city + ret << total + for (def type in types) { + def typeCode = SeaweedAroundPoi.KEYWORDS_MAP.get(type) + def typeCodeArr = typeCode.split("\\|") + String val = "" + String sqlType = "" + if (typeCodeArr[0].size() <= 4) { + val = MyStringUtils.joinLikeSql(typeCodeArr) + sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and (" + val + ") and status <> 1 " + } else { + val = MyStringUtils.joinInSql(typeCodeArr) + sqlType = "select count(*) total from amap_poi where cityname = '" + city + "' and typecode in (" + val + ") and status <> 1 " + } + + ElasticUtils.executeSqlQuery(sqlType,{ data-> + total = data.total + }) + ret << total + } + result << ret + println(city) + } + + ExcelUtils.write("F:\\工作\\房产估值模型及数据招标\\交付工作\\POI信息抓取情况总结.xlsx", result, "POI统计") + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy new file mode 100644 index 000000000..229499b80 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.util + +import com.alibaba.excel.EasyExcelFactory +import com.alibaba.excel.ExcelReader +import com.alibaba.excel.context.AnalysisContext +import com.alibaba.excel.event.AnalysisEventListener + +/** + * excel工具类 + * + * @author shenjian + * @since 2021/5/15 + */ +class ExcelUtils { + + static List> read(String filePath) { + SimpleExcelListener listener = new SimpleExcelListener(filePath); + ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); + excelReader.readAll() + List> dataList = listener.getDataList(); + excelReader.finish() + return dataList; + } + + class SimpleExcelListener extends AnalysisEventListener { + + private List> dataList = new ArrayList<>() + + @Override + void invoke(Object o, AnalysisContext analysisContext) { + Map map = o; + dataList.add(new ArrayList(map.values())) + } + + @Override + void doAfterAllAnalysed(AnalysisContext analysisContext) { + + } + + List> getDataList() { + return dataList + } + + void setDataList(List> dataList) { + this.dataList = dataList + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy new file mode 100644 index 000000000..36eb6c935 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/MyStringUtils.groovy @@ -0,0 +1,40 @@ +package com.fangjia.script.temp.shenjian.util + +/** + * @author shenjian* @since 2021/7/7 + */ +public class MyStringUtils { + + public static joinSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append("'").append(str[i]).append("'") + if (i != str.size() - 1) { + sb.append(",") + } + } + return sb.toString() + } + + public static joinInSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append("'").append(str[i]).append("'") + if (i != str.size() - 1) { + sb.append(",") + } + } + return sb.toString() + } + + public static joinLikeSql(String[] str) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < str.size(); i++) { + sb.append(" typecode like '").append(str[i]).append("%' ") + if (i != str.size() - 1) { + sb.append(" or ") + } + } + return sb.toString() + } +} From 5574322e649f8d30250328d00b1897c532cde34d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 7 Jul 2021 14:05:46 +0800 Subject: [PATCH 083/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dexcel=E5=90=88?= =?UTF-8?q?=E5=B9=B6bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../groovy/com/fangjia/script/util/ExcelUtils.groovy | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy index 203508a51..cc026d9fc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ExcelUtils.groovy @@ -346,11 +346,13 @@ public class ExcelUtils{ mergedCellStyle.alignment = HorizontalAlignment.CENTER; for (List rowCell : data) { - CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3)) - sheet.addMergedRegion(cra); - //设置合并后style - def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2)); - cell.setCellStyle(mergedCellStyle) + if (rowCell.get(0) != rowCell.get(1)) { + CellRangeAddress cra = new CellRangeAddress(rowCell.get(0), rowCell.get(1), rowCell.get(2), rowCell.get(3)) + sheet.addMergedRegion(cra); + //设置合并后style + def cell = sheet.getRow(rowCell.get(0)).getCell(rowCell.get(2)); + cell.setCellStyle(mergedCellStyle) + } } } From 93a2ba322ce17d34bd6aaf6beb9157d3c9ba6070 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 7 Jul 2021 16:22:09 +0800 Subject: [PATCH 084/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=B0=8F=E5=8C=BA?= =?UTF-8?q?=E5=91=A8=E8=BE=B9=E5=B1=85=E5=A7=94=E4=BC=9A=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../poi/SeaweedAroundCommittee.groovy | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy new file mode 100644 index 000000000..d8edf839d --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy @@ -0,0 +1,58 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 小区周边居委会获取 + * + * @author shenjian + * @since 2021/5/24 + */ +class SeaweedAroundCommittee { + + static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + // 乡镇以下级政府及事业单位(居委会) + def types = "130106" + dealPoi(new BasicDBObject(), types) + executorService.shutdown() + } + + static dealPoi(DBObject query, def types) { + query.put("status_committee", 1) + conn136.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def lng = obj.lng + def lat = obj.lat + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 0).append("flag_committee", "已获取居委会"))) + } else { + conn136.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_committee", 2).append("flag_committee", "周边居委会为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} From da6476465147eac5e24fffa6af2de02f8d86c78d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 11:41:50 +0800 Subject: [PATCH 085/212] =?UTF-8?q?=E5=9B=BD=E5=AE=B6=E6=A0=87=E5=87=86?= =?UTF-8?q?=E5=B1=85=E5=A7=94=E4=BC=9A=E7=BB=8F=E7=BA=AC=E5=BA=A6=E8=8E=B7?= =?UTF-8?q?=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/CommitteeLatLng.groovy | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy new file mode 100644 index 000000000..cadb279a7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -0,0 +1,50 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 国家标准居委会经纬度获取 + * + * @author shenjian + * @since 2021/7/22 + */ +class CommitteeLatLng { + + static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + villagesConn.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city_name + def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; + def location = GaoDeUtils.parserByGaode(city, address); + if (location && location.lng) { + def lng = location.lng + def lat = location.lat + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("lng", lng).append("lat", lat).append("flag_location", "已获取经纬度"))) + } else { + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} From 997e10c1d87de8990d57934e5a7329c05deb99f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 11:48:04 +0800 Subject: [PATCH 086/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 002e6aa14..9a3bc83d4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -176,7 +176,7 @@ class GaoDeUtils { */ static def parserByGaode(def city, def address) { JSONObject json = GaoDeApiUtils.geo(city, address) - if (json && json.status == "1") { + if (json && json.info && json.info == "OK") { def ret = json.geocodes if (ret) { def str = ret.get(0).location @@ -184,6 +184,14 @@ class GaoDeUtils { def arrs = str.split(",") return ["lng": arrs[0], "lat": arrs[1]] } + } else if (json && json.infocode && json.infocode == "10044") { + System.exit(1) + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) } } } From 469f9a22f9fd2ab552ebf77d023db03dd186573c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 14:36:35 +0800 Subject: [PATCH 087/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=AB=98=E5=BE=B7?= =?UTF-8?q?=E8=B0=83=E7=94=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/util/GaoDeUtils.groovy | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 9a3bc83d4..4d7cee9f2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.temp.shenjian.util import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil @@ -175,7 +176,17 @@ class GaoDeUtils { * 获取地址经纬度 */ static def parserByGaode(def city, def address) { - JSONObject json = GaoDeApiUtils.geo(city, address) + Map params = new HashMap() + params.put("key", GAO_DE_AK[keyUsed.intValue()]) + params.put("city", city) + params.put("address", address) + String content + try { + content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("https://restapi.amap.com/v3/geocode/geo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) if (json && json.info && json.info == "OK") { def ret = json.geocodes if (ret) { From 2c24a623a7afc98a0389cbfb5d747c507e8d6d55 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 22 Jul 2021 17:05:10 +0800 Subject: [PATCH 088/212] =?UTF-8?q?=E5=B1=85=E5=A7=94=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/CommitteeGeoHash.groovy | 44 +++++++++++++++++++ .../temp/shenjian/poi/CommitteeLatLng.groovy | 27 ++++++++++-- 2 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy new file mode 100644 index 000000000..53b0679f0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeGeoHash.groovy @@ -0,0 +1,44 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 国家标准居委会经纬度geohash化 + * + * @author shenjian + * @since 2021/7/22 + */ +class CommitteeGeoHash { + + static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static void main(String[] args) { + dealPoi(new BasicDBObject("status_location", 0).append("lng", new BasicDBObject('$ne', null))) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + villagesConn.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + double lng = Double.valueOf(obj.lng) + double lat = Double.valueOf(obj.lat) + def geohash = SeaweedUtils.encodeByGeoHash(lng, lat, 8); + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("geohash", geohash))) + println("已处理小区:" + obj) + } + }) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index cadb279a7..1f5f696cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -22,6 +22,7 @@ class CommitteeLatLng { static void main(String[] args) { dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) + //dealLocation() executorService.shutdown() } @@ -32,13 +33,16 @@ class CommitteeLatLng { @Override void run() { def city = obj.city_name + if (city == "市辖区") { + city = obj.province_name + } def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; def location = GaoDeUtils.parserByGaode(city, address); if (location && location.lng) { - def lng = location.lng - def lat = location.lat + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) - .append("lng", lng).append("lat", lat).append("flag_location", "已获取经纬度"))) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度"))) } else { villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) } @@ -47,4 +51,21 @@ class CommitteeLatLng { }) } } + + static dealLocation() { + villagesConn.find(new BasicDBObject("status_location", 0)).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + if (obj.lng && obj.lat) { + def lng = Double.valueOf(obj.lng) + def lat = Double.valueOf(obj.lat) + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("location", new BasicDBObject("lat", lat).append("lon", lng)))) + } + println("已处理小区:" + obj) + } + }) + } + } } From a940ba26ff267037ec872781c9827b4fa94b14a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 4 Aug 2021 13:07:52 +0800 Subject: [PATCH 089/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=94=BF=E5=BA=9C?= =?UTF-8?q?=E6=9C=BA=E5=85=B3=E3=80=81=E4=B8=A7=E8=91=AC=E8=AE=BE=E6=96=BD?= =?UTF-8?q?=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/dingding/DingTalkUtils.groovy | 2 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 25 +++++++++++-------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 9603351a9..209ab9fe2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,7 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-06-01", "yyyy-MM-dd") ; + Date now = DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 00ee277ea..2535bcf4f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -162,17 +162,20 @@ class SeaweedAroundPoi extends BaseBatch { } public static final Map KEYWORDS_MAP = new HashMap(){{ - put("地铁", "150500"); - put("公交", "1507"); - put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); - put("写字楼", "120201"); - put("学校","141200|141201|141202|141203|141204"); - put("银行","1601|1603"); - put("大型超市","0604"); - put("商场","0601"); - put("餐饮","0501|0502|0503|0504"); - put("旅游景点","110000"); - put("图书馆","140500"); +// put("地铁", "150500"); +// put("公交", "1507"); +// put("医院", "090100|090101|090200|090202|090203|090204|090205|090206|090207|090208|090209|090210|090211"); +// put("写字楼", "120201"); +// put("学校","141200|141201|141202|141203|141204"); +// put("银行","1601|1603"); +// put("大型超市","0604"); +// put("商场","0601"); +// put("餐饮","0501|0502|0503|0504"); +// put("旅游景点","110000"); +// put("图书馆","140500"); + put("政府机关","130100|130101|130102|130103|130104|130105|130106|130107"); + put("丧葬设施", "071900|071901|071902|071903") + }}; @Override From 230c825cf588cb066d5ed3dfb550428761ad8572 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 4 Aug 2021 13:21:26 +0800 Subject: [PATCH 090/212] =?UTF-8?q?=E5=85=B7=E4=BD=93=E7=82=B9=E7=BB=8F?= =?UTF-8?q?=E7=BA=AC=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/poi/GetPoi.groovy | 39 ++++++++++++++ .../seaweed/SeaweedGeoHashExport.groovy | 53 +++++++++++-------- .../temp/shenjian/util/SeaweedUtils.groovy | 3 +- 3 files changed, 73 insertions(+), 22 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy new file mode 100644 index 000000000..7ff7cb903 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoi.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.temp.shenjian.poi + + +import com.alibaba.fastjson.JSONObject +import com.fangjia.script.lixue.util.DBUtils +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.temp.shenjian.util.HttpUtils +import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.wenyan.auto.KeywordSearchUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.bson.types.ObjectId + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 获取具体点poi + * + * @author shenjian + * @since 2021/7/26 + */ +class GetPoi { + + static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + + static void main(String[] args) { + List pois = GaoDeUtils.searchNearDtoByPoi("无锡","190301",120.411465,31.864912, 1000, "江苏省江阴市周庄镇兴隆北路"); + if (CollectionUtils.isNotEmpty(pois)) { + String jsonStr = JSONObject.toJSONString(pois) + HttpUtils.postForJson(URL, jsonStr) + } + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy index 154efe43e..a0cede622 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedGeoHashExport.groovy @@ -1,6 +1,9 @@ package com.fangjia.script.temp.shenjian.seaweed +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.SeaweedUtils +import com.fangjia.script.util.DataUtil import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject @@ -14,27 +17,35 @@ import com.mongodb.DBObject class SeaweedGeoHashExport { static void main(String[] args) { - List list = new ArrayList<>() + print(SeaweedUtils.encodeByGeoHash(30.8006287800, 108.3835777247, 8)) + //List> data = com.fangjia.script.util.ExcelUtils.read("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx") +/* for (List str : data) { + String geohash = str.get(0) + str.add(SeaweedUtils.decodeByGeoHash(geohash)) + }*/ - DBObject query = new BasicDBObject() - query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁") - - DBObject filter = new BasicDBObject() - filter.put("lng", 1) - filter.put("lat", 1) - DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp -> - final def obj = objTemp - if (obj.lng && obj.lat) { - def lon = obj.lng - def lat = obj.lat - list.add(SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7)) - } - } - def ret = [] - ret << ['小区geohash'] - for (String val : list) { - ret << [val] - } - com.fangjia.script.util.ExcelUtils.write("F:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); + //com.fangjia.script.util.ExcelUtils.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\银联\\长宁区融合数据结果20210625.xlsx", data, "geohash"); +// DBObject query = new BasicDBObject() +// query.append("cat", "district").append("visible", 0).append("city", "上海").append("region", "长宁") +// +// DBObject filter = new BasicDBObject() +// filter.put("lng", 1) +// filter.put("lat", 1) +// filter.put("name", 1) +// +// def rows = [] +// DbUtils.seaweed.find(query, filter).batchSize(20000).each {objTemp -> +// final def obj = objTemp +// if (obj.lng && obj.lat) { +// def lon = obj.lng +// def lat = obj.lat +// def geohash = SeaweedUtils.encodeByGeoHash(SeaweedUtils.formatLatLng(lon), SeaweedUtils.formatLatLng(lat),7) +// rows << [lon, lat, obj.name, geohash] +// } +// } +// def ret = [] +// ret << ["经度", "纬度", "小区名", "小区geohash"] +// ret.addAll(rows) +// com.fangjia.script.util.ExcelUtils.write("D:\\数据\\基础建设\\geohash.xlsx", ret, "geohash"); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy index 9fa4739f8..f21ff7ae7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/SeaweedUtils.groovy @@ -198,7 +198,8 @@ class SeaweedUtils { * @param args */ static def decodeByGeoHash(String geohash) { - return Geohash.decodeLongitude(geohash) + "," + Geohash.decodeLatitude(geohash) + return DataHandleUtils.getDoubleRound(6, Geohash.decodeLatitude(geohash)) + "," + + DataHandleUtils.getDoubleRound(6, Geohash.decodeLongitude(geohash)) } static void main(String[] args) { From 565acf1eb4e5d6ea4379d150524116c1df366861 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 9 Aug 2021 09:10:28 +0800 Subject: [PATCH 091/212] t --- .../fangjia/script/temp/shenjian/Temp.groovy | 39 ------------------- 1 file changed, 39 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 10e63221e..01d0e126d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,50 +1,11 @@ package com.fangjia.script.temp.shenjian -import com.alibaba.fastjson.JSONArray -import com.alibaba.fastjson.JSONObject -import com.baidubce.http.ApiExplorerClient -import com.baidubce.http.AppSigner -import com.baidubce.http.HttpMethodName -import com.baidubce.model.ApiExplorerRequest -import com.baidubce.model.ApiExplorerResponse -import com.fangjia.script.statistics.poi.AmapPoi -import com.fangjia.script.temp.shenjian.util.SeaweedUtils -import org.elasticsearch.geometry.utils.Geohash - /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - String path = "https://goojia.api.bdymkt.com/estimate"; - ApiExplorerRequest request = new ApiExplorerRequest(HttpMethodName.GET, path); - request.setCredentials("fccf809a0f7748449154af926897aecb", "af1f0088a05c493ab20133ba470f87ef "); - // 设置header参数 - request.addHeaderParameter("Content-Type", "application/json; charset=utf-8"); - - // 设置query参数 - request.addQueryParameter("address", "上海宝山呼玛三村"); - request.addQueryParameter("city", "上海"); - request.addQueryParameter("size", "100"); - - - ApiExplorerClient client = new ApiExplorerClient(new AppSigner()); - - try { - ApiExplorerResponse response = client.sendRequest(request); - // 返回结果格式为Json字符串 - System.out.println(response.getResult()); - } catch (Exception e) { - e.printStackTrace(); - } } - -// 万友七季城C区 经度纬度不同106.569161 29.498502 原始106.56881 29.50058 距离231.7003米 -// 马道子 经度纬度不同106.384087 29.66009 原始106.450656 29.550612 距离12306.7728米 -// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 -// 滨江国际 经度纬度不同109.466428 31.01454 原始109.46799 31.016907 距离308.3405米 -// 澳海澜庭 经度纬度不同109.389067 31.020248 原始109.389069 31.02025 距离0.3029米 -// 富洲新城 经度纬度不同106.474475 29.537152 原始106.420021 29.537655 距离1836.9266米 } From 362049c9e343e1301916a5b4507747e1dbe29581 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 16 Aug 2021 15:48:46 +0800 Subject: [PATCH 092/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AD=A6=E6=A0=A1?= =?UTF-8?q?=E8=8E=B7=E5=8F=96=E7=BB=8F=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SchoolLatLng.groovy | 65 +++++++++++++++++++ .../temp/shenjian/util/ExcelUtils.groovy | 28 ++++---- 2 files changed, 79 insertions(+), 14 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy new file mode 100644 index 000000000..219dd491c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.alibaba.excel.EasyExcel +import com.fangjia.script.temp.shenjian.util.ExcelUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 学校经纬度获取 + * + * @author shenjian + * @since 2021/8/16 + */ +class SchoolLatLng { + + static ExecutorService executorService = Executors.newFixedThreadPool(30) + + static String name = "大连市义务教育学校名录test"; + + static void main(String[] args) { + List> schools = ExcelUtils.read(new FileInputStream("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + ".xlsx")); + Map> result = new ConcurrentHashMap<>(); + CountDownLatch latch = new CountDownLatch(schools.size()); + + for (List schoolTemp : schools) { + final List school = schoolTemp + executorService.submit(new Runnable() { + @Override + void run() { + try { + def city = "大连" + def list = new ArrayList(); + def address = school.get(4) + def location = GaoDeUtils.parserByGaode(city, address); + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + list.add(lng) + list.add(lat) + result.put(address, list) + } + println("已处理:" + address) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + latch.countDown() + } + } + }) + } + latch.await() + + List> head = new ArrayList<>() + for (List school : schools) { + def address = school.get(4) + school.addAll(result.get(address)) + } + EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("小学").doWrite(schools) + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy index 229499b80..fa0927eee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -13,36 +13,36 @@ import com.alibaba.excel.event.AnalysisEventListener */ class ExcelUtils { - static List> read(String filePath) { - SimpleExcelListener listener = new SimpleExcelListener(filePath); - ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); - excelReader.readAll() + public static List> read(InputStream inputStream) { + SimpleExcelListener listener = new SimpleExcelListener(); + ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener) + .headRowNumber(1).build(); + excelReader.readAll(); List> dataList = listener.getDataList(); - excelReader.finish() + excelReader.finish(); return dataList; } - class SimpleExcelListener extends AnalysisEventListener { - - private List> dataList = new ArrayList<>() + public static class SimpleExcelListener extends AnalysisEventListener { + private List> dataList = new ArrayList<>(); @Override - void invoke(Object o, AnalysisContext analysisContext) { - Map map = o; - dataList.add(new ArrayList(map.values())) + public void invoke(Object o, AnalysisContext analysisContext) { + Map map = (Map) o; + dataList.add(new ArrayList<>(map.values())); } @Override - void doAfterAllAnalysed(AnalysisContext analysisContext) { + public void doAfterAllAnalysed(AnalysisContext analysisContext) { } List> getDataList() { - return dataList + return dataList; } void setDataList(List> dataList) { - this.dataList = dataList + this.dataList = dataList; } } } From daf23e52c5d7b346a2b33f7134e834b30ffedb8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 16 Aug 2021 16:18:24 +0800 Subject: [PATCH 093/212] =?UTF-8?q?easyexcel=E4=BE=9D=E8=B5=96poi=E7=89=88?= =?UTF-8?q?=E6=9C=AC3.17?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 871548bdf..2e06fc2a1 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -366,12 +366,12 @@ org.apache.poi poi - 3.10-FINAL + 3.17 org.apache.poi poi-ooxml - 3.10-FINAL + 3.17 xml-apis From 347ef3f4db3728a71c86a4c4e40cfc128f537b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Wed, 18 Aug 2021 14:09:39 +0800 Subject: [PATCH 094/212] =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/SchoolLatLng.groovy | 31 ++++++---- .../temp/shenjian/util/ExcelUtils.groovy | 56 +++++++++++++++++++ .../temp/shenjian/util/GaoDeUtils.groovy | 56 +++++++++++++++++++ 3 files changed, 132 insertions(+), 11 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy index 219dd491c..08dc2c38d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SchoolLatLng.groovy @@ -1,6 +1,8 @@ package com.fangjia.script.temp.shenjian.poi import com.alibaba.excel.EasyExcel +import com.fangjia.script.coredata.util.CollectionUtils +import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.ExcelUtils import com.fangjia.script.temp.shenjian.util.GaoDeUtils @@ -34,14 +36,19 @@ class SchoolLatLng { try { def city = "大连" def list = new ArrayList(); - def address = school.get(4) - def location = GaoDeUtils.parserByGaode(city, address); - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - list.add(lng) - list.add(lat) - result.put(address, list) + def address = school.get(1) + def pois = GaoDeUtils.searchPoi(city, address); + if (CollectionUtils.isNotEmpty(pois)) { + for (AmapPoi amapPoi : pois) { + if (address.contains(amapPoi.getName())) { + String[] locations = amapPoi.getLocation().split(",") + list.add(locations[0]) + list.add(locations[1]) + list.add(amapPoi.getId()) + result.put(address, list) + break; + } + } } println("已处理:" + address) } catch(Exception ex) { @@ -56,10 +63,12 @@ class SchoolLatLng { List> head = new ArrayList<>() for (List school : schools) { - def address = school.get(4) - school.addAll(result.get(address)) + def address = school.get(1) + if (result.contains(address)) { + school.addAll(result.get(address)) + } } - EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("小学").doWrite(schools) + EasyExcel.write("D:\\Users\\Domi\\Desktop\\房价网\\项目\\模型\\学校\\" + name + "结果.xlsx").head(head).sheet("学校").doWrite(schools) executorService.shutdown() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy index fa0927eee..639cd37a9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/ExcelUtils.groovy @@ -1,9 +1,13 @@ package com.fangjia.script.temp.shenjian.util +import com.alibaba.excel.EasyExcel import com.alibaba.excel.EasyExcelFactory import com.alibaba.excel.ExcelReader import com.alibaba.excel.context.AnalysisContext import com.alibaba.excel.event.AnalysisEventListener +import com.alibaba.excel.read.metadata.ReadSheet + +import java.util.concurrent.ConcurrentHashMap /** * excel工具类 @@ -13,6 +17,45 @@ import com.alibaba.excel.event.AnalysisEventListener */ class ExcelUtils { + + static List> read(String filePath) { + SimpleExcelListener listener = new SimpleExcelListener(filePath); + ExcelReader excelReader = EasyExcelFactory.read(filePath, null, listener).headRowNumber(0).build(); + excelReader.readAll() + List> dataList = listener.getDataList(); + excelReader.finish() + return dataList; + } + + static List> readSheet(String fileName, String[] sheetNames) { + Map>> result = new ConcurrentHashMap<>(); + ExcelReader excelReader = EasyExcel.read(fileName).build(); + + // 每个sheet分别创建监听器 + SimpleExcelListener listener0 = new SimpleExcelListener(); + SimpleExcelListener listener1 = new SimpleExcelListener(); + + ReadSheet readSheet0 = EasyExcel.readSheet(0).registerReadListener(listener0).build(); + + ReadSheet readSheet1 = EasyExcel.readSheet(1).registerReadListener(listener1).build(); + + // 这里注意 一定要把sheet1 sheet2 一起传进去,不然有个问题就是03版的excel 会读取多次,浪费性能 + // 可以传多个ReadSheet对象,读多个sheet + excelReader.read(readSheet0, readSheet1); + + // 这里千万别忘记关闭,读的时候会创建临时文件,到时磁盘会崩的 + excelReader.finish(); + + ArrayList listenerList = new ArrayList<>(); + listenerList.add(listener0); + listenerList.add(listener1); + + for (SimpleExcelListener listener : listenerList) { + result.put(listener.getSheetName(), listener.getDataList()) + } + return result + } + public static List> read(InputStream inputStream) { SimpleExcelListener listener = new SimpleExcelListener(); ExcelReader excelReader = EasyExcelFactory.read(inputStream, null, listener) @@ -25,6 +68,15 @@ class ExcelUtils { public static class SimpleExcelListener extends AnalysisEventListener { private List> dataList = new ArrayList<>(); + private String sheetName; + + public SimpleExcelListener() { + + } + + public SimpleExcelListener(String sheetName) { + this.sheetName = sheetName + } @Override public void invoke(Object o, AnalysisContext analysisContext) { @@ -44,5 +96,9 @@ class ExcelUtils { void setDataList(List> dataList) { this.dataList = dataList; } + + public String getSheetName() { + return sheetName + } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 4d7cee9f2..1c0188047 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -165,6 +165,62 @@ class GaoDeUtils { return result } + /** + * 查找关键字经纬度 + */ + static List searchPoi(String city, String keywords){ + def params = [:] + params.key = GAO_DE_AK[keyUsed.intValue()] + params.keywords = keywords + params.city = city + params.put("extensions", "all") + params.put("citylimit","true") + params.put("offset","20") + List result = [] + + int page = 1 + int count = 10000 + while (result.size() < count) { + params.page = "${page}".toString() + page++ + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + if (json == null) { + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + } + if (json && json.info && json.info == "OK") { + count = json.getString("count").toInteger() + try { + List ret = JSONObject.parseArray(json.pois.toString(), AmapPoi.class) + result.addAll(ret) + } catch(Exception ex) { + ex.printStackTrace() + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + break; + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + break; + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } else { + break + } + } + return result + } + /** * 获取经纬度之间距离 */ From ca55e1a30e91be3e991f9d59ad607957b2297e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 2 Sep 2021 19:08:26 +0800 Subject: [PATCH 095/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=B5=A6=E4=B8=9C?= =?UTF-8?q?=E5=AF=B9=E5=8F=A3=E5=B0=8F=E5=AD=A6=E5=88=9D=E4=B8=AD=E6=8A=93?= =?UTF-8?q?=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/SeaweedSchool.groovy | 159 +++++++++++ .../script/statistics/json/housedetail.json | 42 +++ .../script/statistics/json/houselist.json | 255 ++++++++++++++++++ .../script/statistics/json/school.json | 65 +++++ .../statistics/seaweed/HouseDetail.java | 79 ++++++ .../script/statistics/seaweed/HouseList.java | 107 ++++++++ .../script/statistics/seaweed/School.java | 85 ++++++ 7 files changed, 792 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy new file mode 100644 index 000000000..09131cfb4 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -0,0 +1,159 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.Constants +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.MongoCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.jdom.Content; + +/** + * @author shenjian* @since 2021/9/2 + */ +class SeaweedSchool { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.mapmapping.com") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("Content-Type", "application/json") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Referer", "https://servicewechat.com/wx5568ec264e167aeb/9/page-frame.html") + headers.put("Accept-Encoding", "gzip, deflate, br") + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + final String area = "浦东新区"; + final int perPage = 10 + final int total = 4733; + for (int page = 1; page < total / perPage; page++) { + Thread.sleep(10000) + print("当前页:" + page) + try { + Map params = new HashMap(); + params.put("city", "上海市") + params.put("area", area) + params.put("page", String.valueOf(page)) + params.put("perPage", String.valueOf(perPage)) + params.put("priceType", "") + params.put("order", "") + params.put("source", "1") + str = re.get("https://mapmapping.com/school-web/house/getHouseInfoList", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + println str + if (StringUtils.isNotEmpty(str)) { + HouseList houseList = JSONObject.parseObject(str, HouseList.class); + if (houseList != null && "000".equals(houseList.getCode())) { + /*Integer total = houseList.getContent().getTotal();*/ + List listDTOS = houseList.getContent().getList(); + if (CollectionUtils.isNotEmpty(listDTOS)) { + for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { + Thread.sleep(5000) + Integer id = listDTO.getId(); + //# 房屋详情 + HouseDetail houseDetail = getHouseDetail(re, id, headers) + //# 对口小学 + School primarySchool = getPrimarySchool(re, id, headers) + //# 对口中学 + School middleSchool = getMiddleSchool(re, id, headers) + String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) + saveData(houseDetail, primarySchool, middleSchool, url); + } + } + } + } + } + } + + private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) { + DBObject dbObject = new BasicDBObject() + dbObject.put("url", url) + dbObject.put("root", "沪上学区房") + dbObject.put("c_date", new Date()) + dbObject.put("city", "上海") + dbObject.put("region", houseDetail.getContent().getArea()) + dbObject.put("name", houseDetail.getContent().getName()) + dbObject.put("address", houseDetail.getContent().getAddress()) + List primarySchools = primarySchool.getContent(); + if (CollectionUtils.isNotEmpty(primarySchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : primarySchools) { + DBObject schoolObject = new BasicDBObject(); + if ("小学".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("primarySchool", schools) + } + + List middleSchools = middleSchool.getContent(); + if (CollectionUtils.isNotEmpty(middleSchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : middleSchools) { + DBObject schoolObject = new BasicDBObject(); + if ("初中".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("middleSchool", schools) + } + + + collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) + } + + private static HouseDetail getHouseDetail(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getHouseInfoDetail", params, headers, "UTF-8"); + return JSONObject.parseObject(str, HouseDetail.class) + } + + private static School getPrimarySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "2") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + + private static School getMiddleSchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "3") + params.put("page", "1") + params.put("perPage", "4") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json new file mode 100644 index 000000000..64af63aad --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json @@ -0,0 +1,42 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "id": 59151, + "name": "凌高小区", + "imgsUrl": "[\"https://image1.ljcdn.com/hdic-resblock/5f42d5a7-acd9-4a6f-a9f9-92dd651adf6c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ff4c17ac-759a-4e52-bf24-561fa02e851a.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e1ace69b-ce4b-43f9-9e5d-b14244d510cc.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/359a3b16-9485-49ef-b1eb-2cae27751622.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0a17322f-4cdd-43b0-aec0-2d6114a5d636.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/174aac59-4da4-4e88-ad6c-d76a17a59e17.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a6c09e7a-8e96-48c3-b291-20603d61355f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ac1cb228-197a-4e60-b0b0-1bc1c451d00e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e4e9ec1b-5430-4269-b5f7-eb934e189e85.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d9e874a1-df15-42de-b5e7-fb369db97723.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e620cc27-d24b-4cf1-ab4e-d6b3467c180d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b3c00e2e-bacc-4683-abab-8da0a6395972.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b77c7777-41cd-44f0-969f-7d7083d402fe.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/712a3c9a-0d43-410b-becb-ebe47e62fc6d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c44cee38-afbe-42b3-bdf7-f3875a5d0837.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/288933d6-5f8c-4510-8396-15acd8f4580c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c1611809-a633-470c-871c-3a2862e50c79.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/07096876-dd38-415a-ac95-27b79452f664.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae7ffa23-426e-42b7-a509-1a6752c95b72.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0fa04835-9077-4ab6-9e24-72955db66906.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/eb7e289a-8e03-4067-81fa-c633d1a429a5.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/deeade88-3b91-4702-9fdf-68b556ec9921.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/75270a55-bef0-4274-8275-825abae9dce3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0dec445b-f47f-4f2f-996f-c77915d52673.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f05af36d-af7e-4d3f-a8ea-8f23536d2d23.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/40fa734e-682d-4ef6-8094-37e5d281a75f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a3e582a6-19d0-4eeb-9735-2a5a7916187e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f56baa4b-d531-4782-a619-9b22a44b8663.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cc14001f-08ce-4769-9c14-ef4ce1e1f406.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/1153dabe-4ecc-4246-aad2-23ac4575f67f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0712ae4-63dc-477f-9335-1e7147c201d1.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4051310b-ee71-489c-af36-dbb2dc8dd4e0.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/272b80dd-5356-4e7a-beaa-c9d2908d6214.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c238f4da-acc3-4abd-bd62-4c43d59aa46b.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae234e8e-d1d7-4a96-ba27-a333364ae907.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7dec0fa6-508b-4220-b20b-77560fe72c03.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/70cbd711-3bd8-4e6b-9b5e-a13ca6b41cfb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6a258444-96ee-4200-820a-449bb611c801.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0983610c-8ead-472c-a70b-9c50ede9cfd8.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/461dbab6-9cc2-4436-a466-3a06fb1bfe1e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7df6667a-948c-435b-81cb-c1c798614b2e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/97feda14-4672-481f-b127-c28b185cd0ca.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a131d9ca-efd3-4609-a43c-d32978d99cf3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/efd8958c-2108-4660-b8c1-611dad53abf2.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/93c57a6a-0832-4ea0-8512-fcf0d9f5d751.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0dbd942-0ce3-45e9-b1f9-f0af683db82e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b492aeef-172e-40ab-b849-ab2c633d9d13.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/87bde5bd-762b-4b47-ad13-9c1e4e083491.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cb526f64-6bc2-4960-bd71-d650e55123f7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/8b18255e-732e-425d-819b-a753a6191174.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6e45cfc3-cff9-4f47-ac1a-6d9525e2c883.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0463904e-94a1-4720-af81-08c5b289cbeb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/2e518748-1f82-41d1-8bd1-0628830456a7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ad79921b-4aac-4f54-bac9-59a624c95be9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/046e791a-e361-4383-8b6c-609fb1b6ca42.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b219aa06-e7bc-4649-ba96-03996a1565d9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d2e5e0da-7722-4f96-ad68-07e83bf3de5d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4ccfea7c-6c44-4f6f-a900-3aaa5354c356.jpg.710x400.jpg\"]", + "area": "浦东", + "town": "洋泾", + "price": "6.64w", + "priceDesc": "二手挂牌均价", + "plotType": "塔楼", + "leaseNum": "4", + "sellNum": "11", + "dealNum": "0", + "buildTime": "1993年建成", + "houseNum": "1344户", + "building": "6栋", + "propertyFee": "0.75元/平米/月", + "propertyCompany": "上海益镇物业管理有限公司", + "developers": "上海建工房产有限公司", + "lat": 31.247013798320733, + "lng": 121.54954132423184, + "imgArr": [ + "https://img.mapmapping.com/b9bb10e4-d216-4fe3-ac8d-0f3d778e288b.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/28582f65-a02a-47df-b730-9f7f5a5851ee.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/5997bbc0-2322-4d93-9714-2d4513db7677.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/4cd79c3a-f543-4a94-89e8-e80241ff7876.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/3b517fea-b3d3-4b1e-8f58-50e4775d7cfb.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/489d39ee-73ee-4d47-9f76-151c4313d5c3.png?imageView2/2/w/700/h/500/q/75" + ], + "focus": false, + "tags": [ + "浦东", + "洋泾", + "塔楼" + ], + "view": 15312, + "address": "上海市浦东新区栖山路401弄-11号-临" + }, + "timestamp": 1630576612732 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json new file mode 100644 index 000000000..9946d0550 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json @@ -0,0 +1,255 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "total": 4733, + "list": [ + { + "id": 59151, + "name": "凌高小区", + "area": "浦东", + "img": "https://img.mapmapping.com/a68e7f60-a8d8-4e25-ade8-2d77c86562cd.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "11", + "leaseNum": "4", + "price": "6.64w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.25306680262166000", + "lng": "121.55601323149715000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.53w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58324, + "name": "梅园四街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/775871a1-e5a9-4e01-88ed-a39f040b51ac.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "15", + "leaseNum": "11", + "price": "13.42w", + "formatPrice": null, + "year": "1988年建成", + "lat": "31.23754371649258600", + "lng": "121.53281691712684000", + "tags": [ + "近地铁2号线东昌路站" + ], + "plotType": "塔楼/板楼/塔板结合", + "buildTime": "1988年建成", + "view": "1.52w", + "tag": "近地铁2号线东昌路站" + }, + { + "id": 59165, + "name": "中星海上景庭", + "area": "浦东", + "img": "https://img.mapmapping.com/4f9d8c2a-8b89-49a8-b45f-ecaff4f3f588.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "8", + "leaseNum": "3", + "price": "10.86w", + "formatPrice": null, + "year": "2004年建成", + "lat": "31.24523893361267700", + "lng": "121.55501162085352000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "板楼", + "buildTime": "2004年建成", + "view": "1.50w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58477, + "name": "海怡别墅(公寓)", + "area": "浦东", + "img": "https://img.mapmapping.com/f50ed86d-474f-47c5-8f83-7c203c510d92.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "0", + "leaseNum": "0", + "price": "7.32w", + "formatPrice": null, + "year": "1994年建成", + "lat": "31.23963538757626600", + "lng": "121.53779361948267000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1994年建成", + "view": "1.50w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 59148, + "name": "崮山小区一街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/b651bbba-97ba-4ce0-831c-925e80b4459f.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范附属小学羽山校区", + "sellerNum": "10", + "leaseNum": "5", + "price": "11.81w", + "formatPrice": null, + "year": "1999年建成", + "lat": "31.24282189657667000", + "lng": "121.56992501943853000", + "tags": [ + "近地铁9号线芳甸路站" + ], + "plotType": "板楼", + "buildTime": "1999年建成", + "view": "1.50w", + "tag": "近地铁9号线芳甸路站" + }, + { + "id": 55895, + "name": "证大家园(四期)", + "area": "浦东", + "img": "https://img.mapmapping.com/d7d0671f-b0ed-4082-a9f6-1883eca3ac85.png?imageView2/2/w/240/h/200/q/75", + "town": "高行", + "schoolTag": "浦东新区福山证大外国语小学", + "sellerNum": "11", + "leaseNum": "7", + "price": "9.63w", + "formatPrice": null, + "year": "2006年建成", + "lat": "31.28831283871331500", + "lng": "121.60787797808233000", + "tags": [ + "近地铁6号线巨峰路站" + ], + "plotType": "板楼", + "buildTime": "2006年建成", + "view": "1.49w", + "tag": "近地铁6号线巨峰路站" + }, + { + "id": 58709, + "name": "新光大楼", + "area": "浦东", + "img": "https://img.mapmapping.com/f8e1dffb-fa85-457b-8a97-fb019239bdf6.png?imageView2/2/w/240/h/200/q/75", + "town": "潍坊", + "schoolTag": "浦东新区明珠小学A区", + "sellerNum": "3", + "leaseNum": "2", + "price": "11.95w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.23234474870607700", + "lng": "121.52462643697852000", + "tags": [ + "近地铁9号线商城路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.49w", + "tag": "近地铁9号线商城路站" + }, + { + "id": 59322, + "name": "盛世年华", + "area": "浦东", + "img": "https://img.mapmapping.com/83727155-29f5-47d9-b54d-4042d9e625ec.png?imageView2/2/w/240/h/200/q/75", + "town": "源深", + "schoolTag": "浦东新区福山花园外国语小学", + "sellerNum": "4", + "leaseNum": "11", + "price": "14.94w", + "formatPrice": null, + "year": "2005年建成", + "lat": "31.23494392135101000", + "lng": "121.54545194344688000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "2005年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58326, + "name": "梅园七街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/30cf630f-830b-4e43-8dc5-648bb7dfb760.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "2", + "leaseNum": "2", + "price": "11.13w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24378516838488700", + "lng": "121.53455423998314000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58599, + "name": "陈家门小区", + "area": "浦东", + "img": "https://img.mapmapping.com/7a9f43ef-67c4-4168-bf24-0190689d331d.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区梅园小学", + "sellerNum": "3", + "leaseNum": "1", + "price": "9.27w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24726096884606800", + "lng": "121.53135690118148000", + "tags": [ + "近地铁4号线浦东大道站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁4号线浦东大道站" + } + ], + "pageNum": 1, + "pageSize": 10, + "size": 10, + "startRow": 1, + "endRow": 10, + "pages": 474, + "prePage": 0, + "nextPage": 2, + "isFirstPage": true, + "isLastPage": false, + "hasPreviousPage": false, + "hasNextPage": true, + "navigatePages": 8, + "navigatepageNums": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ], + "navigateFirstPage": 1, + "navigateLastPage": 8 + }, + "timestamp": 1630575199849 +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json new file mode 100644 index 000000000..125089046 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json @@ -0,0 +1,65 @@ +{ + "code": "000", + "desc": "操作成功", + "content": [ + { + "id": 272113, + "name": "第六师范学校第二附属小学西校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "小学", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "41", + "sellNum": "188", + "plotPrice": "8.69", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + }, + { + "id": 273287, + "name": "进才中学北校苗圃路校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "初中", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "85", + "sellNum": "339", + "plotPrice": "9.33", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + } + ], + "timestamp": 1630576702404 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java new file mode 100644 index 000000000..b02953be7 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java @@ -0,0 +1,79 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseDetail { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "imgsUrl") + private String imgsUrl; + @JSONField(name = "area") + private String area; + @JSONField(name = "town") + private String town; + @JSONField(name = "price") + private String price; + @JSONField(name = "priceDesc") + private String priceDesc; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "dealNum") + private String dealNum; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "houseNum") + private String houseNum; + @JSONField(name = "building") + private String building; + @JSONField(name = "propertyFee") + private String propertyFee; + @JSONField(name = "propertyCompany") + private String propertyCompany; + @JSONField(name = "developers") + private String developers; + @JSONField(name = "lat") + private Double lat; + @JSONField(name = "lng") + private Double lng; + @JSONField(name = "imgArr") + private List imgArr; + @JSONField(name = "focus") + private Boolean focus; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "view") + private Integer view; + @JSONField(name = "address") + private String address; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java new file mode 100644 index 000000000..cd49ae026 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java @@ -0,0 +1,107 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseList { + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "total") + private Integer total; + @JSONField(name = "list") + private List list; + @JSONField(name = "pageNum") + private Integer pageNum; + @JSONField(name = "pageSize") + private Integer pageSize; + @JSONField(name = "size") + private Integer size; + @JSONField(name = "startRow") + private Integer startRow; + @JSONField(name = "endRow") + private Integer endRow; + @JSONField(name = "pages") + private Integer pages; + @JSONField(name = "prePage") + private Integer prePage; + @JSONField(name = "nextPage") + private Integer nextPage; + @JSONField(name = "isFirstPage") + private Boolean isFirstPage; + @JSONField(name = "isLastPage") + private Boolean isLastPage; + @JSONField(name = "hasPreviousPage") + private Boolean hasPreviousPage; + @JSONField(name = "hasNextPage") + private Boolean hasNextPage; + @JSONField(name = "navigatePages") + private Integer navigatePages; + @JSONField(name = "navigatepageNums") + private List navigatepageNums; + @JSONField(name = "navigateFirstPage") + private Integer navigateFirstPage; + @JSONField(name = "navigateLastPage") + private Integer navigateLastPage; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "area") + private String area; + @JSONField(name = "img") + private String img; + @JSONField(name = "town") + private String town; + @JSONField(name = "schoolTag") + private String schoolTag; + @JSONField(name = "sellerNum") + private String sellerNum; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "price") + private String price; + @JSONField(name = "formatPrice") + private Object formatPrice; + @JSONField(name = "year") + private String year; + @JSONField(name = "lat") + private String lat; + @JSONField(name = "lng") + private String lng; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "view") + private String view; + @JSONField(name = "tag") + private String tag; + } + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java new file mode 100644 index 000000000..1beb5153b --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -0,0 +1,85 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class School { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private List content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "nature") + private Object nature; + @JSONField(name = "city") + private String city; + @JSONField(name = "area") + private String area; + @JSONField(name = "type") + private String type; + @JSONField(name = "quality") + private String quality; + @JSONField(name = "location") + private Object location; + @JSONField(name = "picture") + private String picture; + @JSONField(name = "phone") + private String phone; + @JSONField(name = "distanceNum") + private Object distanceNum; + @JSONField(name = "plotNum") + private String plotNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "plotPrice") + private String plotPrice; + @JSONField(name = "address") + private Object address; + @JSONField(name = "lat") + private Object lat; + @JSONField(name = "lng") + private Object lng; + @JSONField(name = "scope") + private Object scope; + @JSONField(name = "score") + private Object score; + @JSONField(name = "scopeArr") + private Object scopeArr; + @JSONField(name = "scopeTag") + private Object scopeTag; + @JSONField(name = "tag") + private Object tag; + @JSONField(name = "shortName") + private Object shortName; + @JSONField(name = "view") + private Object view; + @JSONField(name = "construct") + private Object construct; + @JSONField(name = "echelon") + private Object echelon; + @JSONField(name = "town") + private Object town; + } +} From 5f37c826743b7ae13ec1cd88b153bcd1989dc6f9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 2 Sep 2021 19:08:26 +0800 Subject: [PATCH 096/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=B5=A6=E4=B8=9C?= =?UTF-8?q?=E5=AF=B9=E5=8F=A3=E5=B0=8F=E5=AD=A6=E5=88=9D=E4=B8=AD=E6=8A=93?= =?UTF-8?q?=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/SeaweedSchool.groovy | 159 +++++++++++ .../script/statistics/json/housedetail.json | 42 +++ .../script/statistics/json/houselist.json | 255 ++++++++++++++++++ .../script/statistics/json/school.json | 65 +++++ .../statistics/seaweed/HouseDetail.java | 79 ++++++ .../script/statistics/seaweed/HouseList.java | 107 ++++++++ .../script/statistics/seaweed/School.java | 85 ++++++ 7 files changed, 792 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java create mode 100644 projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy new file mode 100644 index 000000000..5981472a2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -0,0 +1,159 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.Constants +import com.fangjia.core.util.DataUtil +import com.fangjia.core.util.HttpRequest +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.client.MongoCollection +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils +import org.bson.Document +import org.jdom.Content; + +/** + * @author shenjian* @since 2021/9/2 + */ +class SeaweedSchool { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.mapmapping.com") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("Content-Type", "application/json") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Referer", "https://servicewechat.com/wx5568ec264e167aeb/9/page-frame.html") + headers.put("Accept-Encoding", "gzip, deflate, br") + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + final String area = "浦东新区"; + final int perPage = 10 + final int total = 4733; + for (int page = 15; page < total / perPage; page++) { + Thread.sleep(1000) + print("当前页:" + page) + try { + Map params = new HashMap(); + params.put("city", "上海市") + params.put("area", area) + params.put("page", String.valueOf(page)) + params.put("perPage", String.valueOf(perPage)) + params.put("priceType", "") + params.put("order", "") + params.put("source", "1") + str = re.get("https://mapmapping.com/school-web/house/getHouseInfoList", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + println str + if (StringUtils.isNotEmpty(str)) { + HouseList houseList = JSONObject.parseObject(str, HouseList.class); + if (houseList != null && "000".equals(houseList.getCode())) { + /*Integer total = houseList.getContent().getTotal();*/ + List listDTOS = houseList.getContent().getList(); + if (CollectionUtils.isNotEmpty(listDTOS)) { + for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { + Thread.sleep(1000) + Integer id = listDTO.getId(); + //# 房屋详情 + HouseDetail houseDetail = getHouseDetail(re, id, headers) + //# 对口小学 + School primarySchool = getPrimarySchool(re, id, headers) + //# 对口中学 + School middleSchool = getMiddleSchool(re, id, headers) + String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) + saveData(houseDetail, primarySchool, middleSchool, url); + } + } + } + } + } + } + + private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) { + DBObject dbObject = new BasicDBObject() + dbObject.put("url", url) + dbObject.put("root", "沪上学区房") + dbObject.put("c_date", new Date()) + dbObject.put("city", "上海") + dbObject.put("region", houseDetail.getContent().getArea()) + dbObject.put("name", houseDetail.getContent().getName()) + dbObject.put("address", houseDetail.getContent().getAddress()) + List primarySchools = primarySchool.getContent(); + if (CollectionUtils.isNotEmpty(primarySchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : primarySchools) { + DBObject schoolObject = new BasicDBObject(); + if ("小学".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("primarySchool", schools) + } + + List middleSchools = middleSchool.getContent(); + if (CollectionUtils.isNotEmpty(middleSchools)) { + List schools = new ArrayList<>() + for (School.ContentDTO contentDTO : middleSchools) { + DBObject schoolObject = new BasicDBObject(); + if ("初中".equals(contentDTO.getType())) { + schoolObject.put("name", contentDTO.getName()) + schoolObject.put("quality", contentDTO.getQuality()) + schools.add(schoolObject) + } + } + dbObject.put("middleSchool", schools) + } + + + collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) + } + + private static HouseDetail getHouseDetail(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getHouseInfoDetail", params, headers, "UTF-8"); + return JSONObject.parseObject(str, HouseDetail.class) + } + + private static School getPrimarySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "2") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + + private static School getMiddleSchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "3") + params.put("page", "1") + params.put("perPage", "4") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json new file mode 100644 index 000000000..64af63aad --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/housedetail.json @@ -0,0 +1,42 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "id": 59151, + "name": "凌高小区", + "imgsUrl": "[\"https://image1.ljcdn.com/hdic-resblock/5f42d5a7-acd9-4a6f-a9f9-92dd651adf6c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ff4c17ac-759a-4e52-bf24-561fa02e851a.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e1ace69b-ce4b-43f9-9e5d-b14244d510cc.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/359a3b16-9485-49ef-b1eb-2cae27751622.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0a17322f-4cdd-43b0-aec0-2d6114a5d636.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/174aac59-4da4-4e88-ad6c-d76a17a59e17.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a6c09e7a-8e96-48c3-b291-20603d61355f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ac1cb228-197a-4e60-b0b0-1bc1c451d00e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e4e9ec1b-5430-4269-b5f7-eb934e189e85.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d9e874a1-df15-42de-b5e7-fb369db97723.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/e620cc27-d24b-4cf1-ab4e-d6b3467c180d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b3c00e2e-bacc-4683-abab-8da0a6395972.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b77c7777-41cd-44f0-969f-7d7083d402fe.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/712a3c9a-0d43-410b-becb-ebe47e62fc6d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c44cee38-afbe-42b3-bdf7-f3875a5d0837.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/288933d6-5f8c-4510-8396-15acd8f4580c.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c1611809-a633-470c-871c-3a2862e50c79.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/07096876-dd38-415a-ac95-27b79452f664.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae7ffa23-426e-42b7-a509-1a6752c95b72.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0fa04835-9077-4ab6-9e24-72955db66906.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/eb7e289a-8e03-4067-81fa-c633d1a429a5.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/deeade88-3b91-4702-9fdf-68b556ec9921.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/75270a55-bef0-4274-8275-825abae9dce3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0dec445b-f47f-4f2f-996f-c77915d52673.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f05af36d-af7e-4d3f-a8ea-8f23536d2d23.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/40fa734e-682d-4ef6-8094-37e5d281a75f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a3e582a6-19d0-4eeb-9735-2a5a7916187e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/f56baa4b-d531-4782-a619-9b22a44b8663.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cc14001f-08ce-4769-9c14-ef4ce1e1f406.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/1153dabe-4ecc-4246-aad2-23ac4575f67f.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0712ae4-63dc-477f-9335-1e7147c201d1.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4051310b-ee71-489c-af36-dbb2dc8dd4e0.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/272b80dd-5356-4e7a-beaa-c9d2908d6214.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/c238f4da-acc3-4abd-bd62-4c43d59aa46b.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ae234e8e-d1d7-4a96-ba27-a333364ae907.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7dec0fa6-508b-4220-b20b-77560fe72c03.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/70cbd711-3bd8-4e6b-9b5e-a13ca6b41cfb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6a258444-96ee-4200-820a-449bb611c801.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0983610c-8ead-472c-a70b-9c50ede9cfd8.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/461dbab6-9cc2-4436-a466-3a06fb1bfe1e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/7df6667a-948c-435b-81cb-c1c798614b2e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/97feda14-4672-481f-b127-c28b185cd0ca.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/a131d9ca-efd3-4609-a43c-d32978d99cf3.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/efd8958c-2108-4660-b8c1-611dad53abf2.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/93c57a6a-0832-4ea0-8512-fcf0d9f5d751.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b0dbd942-0ce3-45e9-b1f9-f0af683db82e.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b492aeef-172e-40ab-b849-ab2c633d9d13.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/87bde5bd-762b-4b47-ad13-9c1e4e083491.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/cb526f64-6bc2-4960-bd71-d650e55123f7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/8b18255e-732e-425d-819b-a753a6191174.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/6e45cfc3-cff9-4f47-ac1a-6d9525e2c883.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/0463904e-94a1-4720-af81-08c5b289cbeb.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/2e518748-1f82-41d1-8bd1-0628830456a7.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/ad79921b-4aac-4f54-bac9-59a624c95be9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/046e791a-e361-4383-8b6c-609fb1b6ca42.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/b219aa06-e7bc-4649-ba96-03996a1565d9.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/d2e5e0da-7722-4f96-ad68-07e83bf3de5d.jpg.710x400.jpg\",\"https://image1.ljcdn.com/hdic-resblock/4ccfea7c-6c44-4f6f-a900-3aaa5354c356.jpg.710x400.jpg\"]", + "area": "浦东", + "town": "洋泾", + "price": "6.64w", + "priceDesc": "二手挂牌均价", + "plotType": "塔楼", + "leaseNum": "4", + "sellNum": "11", + "dealNum": "0", + "buildTime": "1993年建成", + "houseNum": "1344户", + "building": "6栋", + "propertyFee": "0.75元/平米/月", + "propertyCompany": "上海益镇物业管理有限公司", + "developers": "上海建工房产有限公司", + "lat": 31.247013798320733, + "lng": 121.54954132423184, + "imgArr": [ + "https://img.mapmapping.com/b9bb10e4-d216-4fe3-ac8d-0f3d778e288b.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/28582f65-a02a-47df-b730-9f7f5a5851ee.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/5997bbc0-2322-4d93-9714-2d4513db7677.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/4cd79c3a-f543-4a94-89e8-e80241ff7876.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/3b517fea-b3d3-4b1e-8f58-50e4775d7cfb.png?imageView2/2/w/700/h/500/q/75", + "https://img.mapmapping.com/489d39ee-73ee-4d47-9f76-151c4313d5c3.png?imageView2/2/w/700/h/500/q/75" + ], + "focus": false, + "tags": [ + "浦东", + "洋泾", + "塔楼" + ], + "view": 15312, + "address": "上海市浦东新区栖山路401弄-11号-临" + }, + "timestamp": 1630576612732 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json new file mode 100644 index 000000000..9946d0550 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/houselist.json @@ -0,0 +1,255 @@ +{ + "code": "000", + "desc": "操作成功", + "content": { + "total": 4733, + "list": [ + { + "id": 59151, + "name": "凌高小区", + "area": "浦东", + "img": "https://img.mapmapping.com/a68e7f60-a8d8-4e25-ade8-2d77c86562cd.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "11", + "leaseNum": "4", + "price": "6.64w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.25306680262166000", + "lng": "121.55601323149715000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.53w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58324, + "name": "梅园四街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/775871a1-e5a9-4e01-88ed-a39f040b51ac.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "15", + "leaseNum": "11", + "price": "13.42w", + "formatPrice": null, + "year": "1988年建成", + "lat": "31.23754371649258600", + "lng": "121.53281691712684000", + "tags": [ + "近地铁2号线东昌路站" + ], + "plotType": "塔楼/板楼/塔板结合", + "buildTime": "1988年建成", + "view": "1.52w", + "tag": "近地铁2号线东昌路站" + }, + { + "id": 59165, + "name": "中星海上景庭", + "area": "浦东", + "img": "https://img.mapmapping.com/4f9d8c2a-8b89-49a8-b45f-ecaff4f3f588.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范学校第二附属小学西校区", + "sellerNum": "8", + "leaseNum": "3", + "price": "10.86w", + "formatPrice": null, + "year": "2004年建成", + "lat": "31.24523893361267700", + "lng": "121.55501162085352000", + "tags": [ + "近地铁6号线北洋泾路站" + ], + "plotType": "板楼", + "buildTime": "2004年建成", + "view": "1.50w", + "tag": "近地铁6号线北洋泾路站" + }, + { + "id": 58477, + "name": "海怡别墅(公寓)", + "area": "浦东", + "img": "https://img.mapmapping.com/f50ed86d-474f-47c5-8f83-7c203c510d92.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "0", + "leaseNum": "0", + "price": "7.32w", + "formatPrice": null, + "year": "1994年建成", + "lat": "31.23963538757626600", + "lng": "121.53779361948267000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1994年建成", + "view": "1.50w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 59148, + "name": "崮山小区一街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/b651bbba-97ba-4ce0-831c-925e80b4459f.png?imageView2/2/w/240/h/200/q/75", + "town": "洋泾", + "schoolTag": "第六师范附属小学羽山校区", + "sellerNum": "10", + "leaseNum": "5", + "price": "11.81w", + "formatPrice": null, + "year": "1999年建成", + "lat": "31.24282189657667000", + "lng": "121.56992501943853000", + "tags": [ + "近地铁9号线芳甸路站" + ], + "plotType": "板楼", + "buildTime": "1999年建成", + "view": "1.50w", + "tag": "近地铁9号线芳甸路站" + }, + { + "id": 55895, + "name": "证大家园(四期)", + "area": "浦东", + "img": "https://img.mapmapping.com/d7d0671f-b0ed-4082-a9f6-1883eca3ac85.png?imageView2/2/w/240/h/200/q/75", + "town": "高行", + "schoolTag": "浦东新区福山证大外国语小学", + "sellerNum": "11", + "leaseNum": "7", + "price": "9.63w", + "formatPrice": null, + "year": "2006年建成", + "lat": "31.28831283871331500", + "lng": "121.60787797808233000", + "tags": [ + "近地铁6号线巨峰路站" + ], + "plotType": "板楼", + "buildTime": "2006年建成", + "view": "1.49w", + "tag": "近地铁6号线巨峰路站" + }, + { + "id": 58709, + "name": "新光大楼", + "area": "浦东", + "img": "https://img.mapmapping.com/f8e1dffb-fa85-457b-8a97-fb019239bdf6.png?imageView2/2/w/240/h/200/q/75", + "town": "潍坊", + "schoolTag": "浦东新区明珠小学A区", + "sellerNum": "3", + "leaseNum": "2", + "price": "11.95w", + "formatPrice": null, + "year": "1993年建成", + "lat": "31.23234474870607700", + "lng": "121.52462643697852000", + "tags": [ + "近地铁9号线商城路站" + ], + "plotType": "塔楼", + "buildTime": "1993年建成", + "view": "1.49w", + "tag": "近地铁9号线商城路站" + }, + { + "id": 59322, + "name": "盛世年华", + "area": "浦东", + "img": "https://img.mapmapping.com/83727155-29f5-47d9-b54d-4042d9e625ec.png?imageView2/2/w/240/h/200/q/75", + "town": "源深", + "schoolTag": "浦东新区福山花园外国语小学", + "sellerNum": "4", + "leaseNum": "11", + "price": "14.94w", + "formatPrice": null, + "year": "2005年建成", + "lat": "31.23494392135101000", + "lng": "121.54545194344688000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "2005年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58326, + "name": "梅园七街坊", + "area": "浦东", + "img": "https://img.mapmapping.com/30cf630f-830b-4e43-8dc5-648bb7dfb760.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区福山外国语小学(福山/瑞华)", + "sellerNum": "2", + "leaseNum": "2", + "price": "11.13w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24378516838488700", + "lng": "121.53455423998314000", + "tags": [ + "近地铁6号线源深体育中心站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁6号线源深体育中心站" + }, + { + "id": 58599, + "name": "陈家门小区", + "area": "浦东", + "img": "https://img.mapmapping.com/7a9f43ef-67c4-4168-bf24-0190689d331d.png?imageView2/2/w/240/h/200/q/75", + "town": "梅园", + "schoolTag": "浦东新区梅园小学", + "sellerNum": "3", + "leaseNum": "1", + "price": "9.27w", + "formatPrice": null, + "year": "1986年建成", + "lat": "31.24726096884606800", + "lng": "121.53135690118148000", + "tags": [ + "近地铁4号线浦东大道站" + ], + "plotType": "板楼", + "buildTime": "1986年建成", + "view": "1.48w", + "tag": "近地铁4号线浦东大道站" + } + ], + "pageNum": 1, + "pageSize": 10, + "size": 10, + "startRow": 1, + "endRow": 10, + "pages": 474, + "prePage": 0, + "nextPage": 2, + "isFirstPage": true, + "isLastPage": false, + "hasPreviousPage": false, + "hasNextPage": true, + "navigatePages": 8, + "navigatepageNums": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8 + ], + "navigateFirstPage": 1, + "navigateLastPage": 8 + }, + "timestamp": 1630575199849 +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json new file mode 100644 index 000000000..125089046 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/json/school.json @@ -0,0 +1,65 @@ +{ + "code": "000", + "desc": "操作成功", + "content": [ + { + "id": 272113, + "name": "第六师范学校第二附属小学西校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "小学", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "41", + "sellNum": "188", + "plotPrice": "8.69", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + }, + { + "id": 273287, + "name": "进才中学北校苗圃路校区", + "nature": null, + "city": "上海市", + "area": "浦东新区", + "type": "初中", + "quality": "公办", + "location": null, + "picture": "http://img.mapmapping.com//FpujwY5MscqLxCjjD27Mg-A4MY8d", + "phone": "", + "distanceNum": null, + "plotNum": "85", + "sellNum": "339", + "plotPrice": "9.33", + "address": null, + "lat": null, + "lng": null, + "scope": null, + "score": null, + "scopeArr": null, + "scopeTag": null, + "tag": null, + "shortName": null, + "view": null, + "construct": null, + "echelon": null, + "town": null + } + ], + "timestamp": 1630576702404 +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java new file mode 100644 index 000000000..b02953be7 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseDetail.java @@ -0,0 +1,79 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseDetail { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "imgsUrl") + private String imgsUrl; + @JSONField(name = "area") + private String area; + @JSONField(name = "town") + private String town; + @JSONField(name = "price") + private String price; + @JSONField(name = "priceDesc") + private String priceDesc; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "dealNum") + private String dealNum; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "houseNum") + private String houseNum; + @JSONField(name = "building") + private String building; + @JSONField(name = "propertyFee") + private String propertyFee; + @JSONField(name = "propertyCompany") + private String propertyCompany; + @JSONField(name = "developers") + private String developers; + @JSONField(name = "lat") + private Double lat; + @JSONField(name = "lng") + private Double lng; + @JSONField(name = "imgArr") + private List imgArr; + @JSONField(name = "focus") + private Boolean focus; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "view") + private Integer view; + @JSONField(name = "address") + private String address; + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java new file mode 100644 index 000000000..cd49ae026 --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/HouseList.java @@ -0,0 +1,107 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class HouseList { + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private ContentDTO content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "total") + private Integer total; + @JSONField(name = "list") + private List list; + @JSONField(name = "pageNum") + private Integer pageNum; + @JSONField(name = "pageSize") + private Integer pageSize; + @JSONField(name = "size") + private Integer size; + @JSONField(name = "startRow") + private Integer startRow; + @JSONField(name = "endRow") + private Integer endRow; + @JSONField(name = "pages") + private Integer pages; + @JSONField(name = "prePage") + private Integer prePage; + @JSONField(name = "nextPage") + private Integer nextPage; + @JSONField(name = "isFirstPage") + private Boolean isFirstPage; + @JSONField(name = "isLastPage") + private Boolean isLastPage; + @JSONField(name = "hasPreviousPage") + private Boolean hasPreviousPage; + @JSONField(name = "hasNextPage") + private Boolean hasNextPage; + @JSONField(name = "navigatePages") + private Integer navigatePages; + @JSONField(name = "navigatepageNums") + private List navigatepageNums; + @JSONField(name = "navigateFirstPage") + private Integer navigateFirstPage; + @JSONField(name = "navigateLastPage") + private Integer navigateLastPage; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "area") + private String area; + @JSONField(name = "img") + private String img; + @JSONField(name = "town") + private String town; + @JSONField(name = "schoolTag") + private String schoolTag; + @JSONField(name = "sellerNum") + private String sellerNum; + @JSONField(name = "leaseNum") + private String leaseNum; + @JSONField(name = "price") + private String price; + @JSONField(name = "formatPrice") + private Object formatPrice; + @JSONField(name = "year") + private String year; + @JSONField(name = "lat") + private String lat; + @JSONField(name = "lng") + private String lng; + @JSONField(name = "tags") + private List tags; + @JSONField(name = "plotType") + private String plotType; + @JSONField(name = "buildTime") + private String buildTime; + @JSONField(name = "view") + private String view; + @JSONField(name = "tag") + private String tag; + } + } +} diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java new file mode 100644 index 000000000..1beb5153b --- /dev/null +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -0,0 +1,85 @@ +package com.fangjia.script.statistics.seaweed; + +import com.alibaba.fastjson.annotation.JSONField; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @author shenjian + * @since 2021/9/2 + */ +@NoArgsConstructor +@Data +public class School { + + + @JSONField(name = "code") + private String code; + @JSONField(name = "desc") + private String desc; + @JSONField(name = "content") + private List content; + @JSONField(name = "timestamp") + private Long timestamp; + + @NoArgsConstructor + @Data + public static class ContentDTO { + @JSONField(name = "id") + private Integer id; + @JSONField(name = "name") + private String name; + @JSONField(name = "nature") + private Object nature; + @JSONField(name = "city") + private String city; + @JSONField(name = "area") + private String area; + @JSONField(name = "type") + private String type; + @JSONField(name = "quality") + private String quality; + @JSONField(name = "location") + private Object location; + @JSONField(name = "picture") + private String picture; + @JSONField(name = "phone") + private String phone; + @JSONField(name = "distanceNum") + private Object distanceNum; + @JSONField(name = "plotNum") + private String plotNum; + @JSONField(name = "sellNum") + private String sellNum; + @JSONField(name = "plotPrice") + private String plotPrice; + @JSONField(name = "address") + private Object address; + @JSONField(name = "lat") + private Object lat; + @JSONField(name = "lng") + private Object lng; + @JSONField(name = "scope") + private Object scope; + @JSONField(name = "score") + private Object score; + @JSONField(name = "scopeArr") + private Object scopeArr; + @JSONField(name = "scopeTag") + private Object scopeTag; + @JSONField(name = "tag") + private Object tag; + @JSONField(name = "shortName") + private Object shortName; + @JSONField(name = "view") + private Object view; + @JSONField(name = "construct") + private Object construct; + @JSONField(name = "echelon") + private Object echelon; + @JSONField(name = "town") + private Object town; + } +} From 729b2a246c78f6e3580111eec2fac8cbd5c6418a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 3 Sep 2021 09:20:48 +0800 Subject: [PATCH 097/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=B8=80=E6=89=8B?= =?UTF-8?q?=E6=88=BF=E6=88=90=E4=BA=A4=E4=BF=A1=E6=81=AF=E6=8A=93=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/seaweed/NewHouseFangDi.groovy | 70 +++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy new file mode 100644 index 000000000..b1b702fb6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/NewHouseFangDi.groovy @@ -0,0 +1,70 @@ +package com.fangjia.script.temp.shenjian.seaweed + +import com.alibaba.fastjson.JSONObject +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.statistics.seaweed.HouseDetail +import com.fangjia.script.statistics.seaweed.HouseList +import com.fangjia.script.statistics.seaweed.School +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +/** + * @author shenjian* @since 2021/9/2 + */ +class NewHouseFangDi { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + public static Map headerMap(String cook) { + Map headers = new HashMap(); + headers.put("Host", "www.fangdi.com.cn") + headers.put("Connection", "keep-alive") + headers.put("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36") + headers.put("Content-Type", "application/x-www-form-urlencoded;charset=utf-8") + headers.put("Accept", "application/json, text/javascript, */*; q=0.01") + headers.put("X-Requested-With", "XMLHttpRequest") + headers.put("Origin", "http://www.fangdi.com.cn") + headers.put("Referer", "http://www.fangdi.com.cn/new_house/new_house_list_fast.html?dicPriceID=&dicRegionID=&flatID=&districtID=5391be55e75e5506&check_input2=105C") + headers.put("Cookie", "_applet_id=34185;_applet_ticket=e294e2426345401088a3e828128e70e3") + headers.put("Accept-Encoding", "gzip, deflate") + headers.put("Accept-Language", "zh-CN,zh;q=0.9") + headers.put("Cookie", "www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; www.fangdi.com.cn=www.fangdi.com.cn_rs3; FSSBBIl1UgzbN7N80S=QS7tHXu4vqldPlDeqJkUCGoyqBmqHzTtyqvrk2ibcRKGVcbPOJ7ERuh5qYgHVD4l; www.fangdi.com_http_ic=www.fangdi.com.cn_80_RS; FSSBBIl1UgzbN7Nenable=true; JSESSIONID1=jw-mUJZsTiFYkXShzn6asMa401YoEZs-0WzhNHrUGId4Jh1_14tj!-338668155; FSSBBIl1UgzbN7N80T=4t2lc885DGECHVNGMwL4w1LcI9v43OwswK7ZOXM5ZbcyVF1UvDPUs59KVOQkq_2hxc0iJrZi3gBiildN1Xfh7xNz81qFz12KwlUJrncPg8mejTeav9tTOgiiPXMZZfJ4rltOCYgOzLLqbj5nG9esEse9UWPYnFdkTWvjpJUTfdyIGg9vB_S9ABAvmir3e89JTbzvfK4.i_IRPFED2hsteZcTiIpNWteOxZMMeDPSnKJei2_Hqhmn7LQMVZanibREAJ8Ah7IxIAwcMtRqAtZLLRpcQMtwnqkEoUPHf435yJVKL20fHui11nhVV4MxTHo9pcoz6L.yhuezCyP5NGTdpV14XV_yBCBhNkSXKWbO.lWAEUQ8yBkMwLLgaYY34sg0Bz_0") + + + return headers; + } + + static void main(String[] args) { + Map headers = headerMap("") + HttpRequest re = new HttpRequest(); + String str = null + try { + Map params = new HashMap(); +// params.put("districtID", "5391be55e75e5506") +// params.put("dicRegionID", "") +// params.put("dicPriceID", "") +// params.put("flatID", "") + params.put("districtID", "5391be55e75e5506") + params.put("dicRegionID", "") + params.put("stateID", "4d29bb4241910ce3") + params.put("houseAreaID", "0") + params.put("dicAvgpriceID", "0") + params.put("dicPositionID", "") + params.put("houseTypeID", "0") + params.put("houseTypeID", "0") + params.put("address", "") + params.put("openingID", "") + params.put("projectName", "") + params.put("currentPage", "1") + str = re.post("http://www.fangdi.com.cn/service/freshHouse/getFastHosueList.action", params, headers, "UTF-8"); + } catch (Exception e) { + e.printStackTrace() + } + def data = JSONObject.parseObject(str).get("data") + print(data) + } +} From d78e6a664a840cc0fa34536d01f6640243fd58d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 3 Sep 2021 14:04:54 +0800 Subject: [PATCH 098/212] =?UTF-8?q?=E5=95=86=E5=9C=88=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E6=96=B0=E6=96=B9=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/BusinessLatLng.groovy | 46 +++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy new file mode 100644 index 000000000..519e5de21 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/BusinessLatLng.groovy @@ -0,0 +1,46 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors + +/** + * 商圈经纬度 + */ +class BusinessLatLng { + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_crawl", "business", "192.168.0.235", 27018) + static ExecutorService executorService = Executors.newFixedThreadPool(1) + + static void main(String[] args) { + dealPoi(new BasicDBObject("city", "上海").append("region", "浦东新区")) + executorService.shutdown() + } + + static dealPoi(DBObject query) { + collection.find(query).batchSize(5000).each {objTemp -> + final def obj = objTemp + executorService.submit(new Runnable() { + @Override + void run() { + def city = obj.city + def address = city + obj.region + obj.address; + def location = GaoDeUtils.parserByGaode(city, address); + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度"))) + } else { + collection.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } + println("已处理小区:" + obj) + } + }) + } + } +} From 776b32e1aaa2b576f891bb79f28cc353a4550d2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 3 Sep 2021 14:22:59 +0800 Subject: [PATCH 099/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AF=B9=E5=8F=A3?= =?UTF-8?q?=E7=BB=8F=E7=BA=AC=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/SeaweedSchoolLatLng.groovy | 86 +++++++++++++++++++ .../shenjian/seaweed/SeaweedSchool.groovy | 4 +- .../temp/shenjian/util/GaoDeUtils.groovy | 49 ++++++++++- 3 files changed, 136 insertions(+), 3 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy new file mode 100644 index 000000000..d511bb745 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -0,0 +1,86 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBList +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +/** + * 对口小学初中经纬度 + */ +class SeaweedSchoolLatLng { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + + static ExecutorService executorService = Executors.newFixedThreadPool(20) + + static void main(String[] args) { + AtomicLong atomicLong = new AtomicLong(0); + collection.find(new BasicDBObject()).batchSize(1000).each {objT -> + final DBObject obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def name = dbObject.name + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + if (dbObject.lng != null) { + String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) + dbObject.address = address + } + } + } + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def name = dbObject.name + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + if (dbObject.lng != null) { + String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) + dbObject.address = address + } + } + } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) + } catch(Exception ex) { + ex.printStackTrace() + } finally { + atomicLong.incrementAndGet() + synchronized (this) { + if (atomicLong.longValue() % 100 == 0) { + println(atomicLong.longValue()) + } + } + } + } + }) + }; + executorService.shutdown() + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index 5981472a2..73545a2d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -19,7 +19,9 @@ import org.bson.Document import org.jdom.Content; /** - * @author shenjian* @since 2021/9/2 + * 对口小学、初中 + * @author shenjian + * @since 2021/9/2 */ class SeaweedSchool { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 1c0188047..5334643d3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -4,6 +4,7 @@ import com.alibaba.fastjson.JSONArray import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil import com.fangjia.core.util.HttpRequest +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil @@ -214,8 +215,6 @@ class GaoDeUtils { } else if (json && json.infocode && json.infocode == "10017") { // 所请求的资源不可用 System.exit(1) - } else { - break } } return result @@ -228,6 +227,52 @@ class GaoDeUtils { return GeoHelper.GetDistance(Lat, Lng, Lat1, Lng1) } + /** + * 经纬度转地址
+ * 规则: 最多支持20个坐标点。多个点之间用"|"分割。经度在前,纬度在后,经纬度间以“,”分割,经纬度小数点后不得超过6位 + * @author yinjihuan + * @date 2016年9月9日 + * @param location + * @return + */ + static def getGaoDeAddress(location) { + Map params = new HashMap() + params.put("key", GAO_DE_AK[keyUsed.intValue()]) + params.put("location", location) + String content + try { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } catch (Exception e) { + content = new HttpRequest().get("http://restapi.amap.com/v3/geocode/regeo", params, "utf-8") + } + net.sf.json.JSONObject json = net.sf.json.JSONObject.fromObject(content) + Map infoMap = DataHandleUtils.JsonToMap(json); + if (infoMap && infoMap.info == "OK") { + def regeocode = infoMap.regeocode + if (regeocode) { + return regeocode.formatted_address; + } + } else if (json && json.infocode && json.infocode == "10044") { + synchronized (this) { + for (int i = 0; i < GAO_DE_AK.size(); i++) { + if (GAO_DE_AK[i] == params.key) { + if (i + 1 > GAO_DE_AK.size()) { + System.exit(1) + } + keyUsed = new AtomicInteger(i+1) + } + } + } + } else if (json && json.infocode && json.infocode == "10004") { + // 开发者的单位时间内(1分钟)访问量超限,被系统自动封停,下一分钟自动解封。 + Thread.sleep(60000) + } else if (json && json.infocode && json.infocode == "10017") { + // 所请求的资源不可用 + System.exit(1) + } + return null + } + /** * 获取地址经纬度 */ From f267f393fe99315173cb8077a9e77d8b1ddbdec4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 5 Sep 2021 11:02:26 +0800 Subject: [PATCH 100/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=AF=B9=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/Temp.groovy | 6 +- .../shenjian/seaweed/SeaweedSchool.groovy | 91 +++++++++++----- .../script/statistics/seaweed/School.java | 101 +++++++----------- 3 files changed, 109 insertions(+), 89 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy index 01d0e126d..517fcf4ed 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/Temp.groovy @@ -1,11 +1,15 @@ package com.fangjia.script.temp.shenjian +import com.fangjia.script.temp.shenjian.util.GaoDeUtils + /** * @author shenjian* @since 2021/5/18 */ class Temp { static void main(String[] args) { - + // 121.431870,31.208610 + List list = GaoDeUtils.searchPoi("上海", "浦东新区新场实验小学新城校区") + print(list) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index 73545a2d8..b988b6a9d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -44,10 +44,10 @@ class SeaweedSchool { Map headers = headerMap("") HttpRequest re = new HttpRequest(); String str = null - final String area = "浦东新区"; + final String area = "浦东"; final int perPage = 10 final int total = 4733; - for (int page = 15; page < total / perPage; page++) { + for (int page = 1; page < total / perPage; page++) { Thread.sleep(1000) print("当前页:" + page) try { @@ -71,16 +71,18 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(1000) + Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) + // 对口幼儿园 + School nurserySchool = getNurserySchool(re, id, headers) //# 对口小学 School primarySchool = getPrimarySchool(re, id, headers) //# 对口中学 School middleSchool = getMiddleSchool(re, id, headers) String url = "https://mapmapping.com/school-web/house/getHouseInfoDetail?source=1&id=" + String.valueOf(id) - saveData(houseDetail, primarySchool, middleSchool, url); + saveData(houseDetail, nurserySchool, primarySchool, middleSchool, url); } } } @@ -88,7 +90,7 @@ class SeaweedSchool { } } - private static void saveData(HouseDetail houseDetail, School primarySchool, School middleSchool, def url) { + private static void saveData(HouseDetail houseDetail, School nurserySchool, School primarySchool, School middleSchool, def url) { DBObject dbObject = new BasicDBObject() dbObject.put("url", url) dbObject.put("root", "沪上学区房") @@ -97,38 +99,60 @@ class SeaweedSchool { dbObject.put("region", houseDetail.getContent().getArea()) dbObject.put("name", houseDetail.getContent().getName()) dbObject.put("address", houseDetail.getContent().getAddress()) - List primarySchools = primarySchool.getContent(); + dbObject.put("lng", houseDetail.getContent().getLng()) + dbObject.put("lat", houseDetail.getContent().getLat()) + + List nurserySchools = nurserySchool.getContent().getList(); + if (CollectionUtils.isNotEmpty(nurserySchools)) { + List schools = buildSchool(nurserySchools) + dbObject.put("nurserySchool", schools) + } + + List primarySchools = primarySchool.getContent().getList();; if (CollectionUtils.isNotEmpty(primarySchools)) { - List schools = new ArrayList<>() - for (School.ContentDTO contentDTO : primarySchools) { - DBObject schoolObject = new BasicDBObject(); - if ("小学".equals(contentDTO.getType())) { - schoolObject.put("name", contentDTO.getName()) - schoolObject.put("quality", contentDTO.getQuality()) - schools.add(schoolObject) - } - } + List schools = buildSchool(primarySchools) dbObject.put("primarySchool", schools) } - List middleSchools = middleSchool.getContent(); + List middleSchools = middleSchool.getContent().getList();; if (CollectionUtils.isNotEmpty(middleSchools)) { - List schools = new ArrayList<>() - for (School.ContentDTO contentDTO : middleSchools) { - DBObject schoolObject = new BasicDBObject(); - if ("初中".equals(contentDTO.getType())) { - schoolObject.put("name", contentDTO.getName()) - schoolObject.put("quality", contentDTO.getQuality()) - schools.add(schoolObject) - } - } + List schools = buildSchool(middleSchools) dbObject.put("middleSchool", schools) } - collection.update(new BasicDBObject("url", url).append("root", "沪上学区房"), dbObject, true, false) } + private static List buildSchool(List currSchools) { + List schools = new ArrayList<>() + for (School.ContentDTO.ListDTO contentDTO : currSchools) { + DBObject schoolObject = new BasicDBObject(); + if (com.fangjia.script.coredata.util.CollectionUtils.isNotEmpty(contentDTO.getTags())) { + if (contentDTO.getTags().size() >= 3) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", contentDTO.getTags().get(2)) + } else if (contentDTO.getTags().size() == 2) { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", contentDTO.getTags().get(1)) + schoolObject.put("batch", '') + } else { + schoolObject.put("quality", contentDTO.getTags().get(0)) + schoolObject.put("type", '') + schoolObject.put("batch", '') + } + } + schoolObject.put("area", contentDTO.getArea()) + schoolObject.put("town", contentDTO.getTown()) + schoolObject.put("distance", contentDTO.getDistance()) + schoolObject.put("address", contentDTO.getAddress()) + schools.add(schoolObject) + } + return schools + } + + + private static HouseDetail getHouseDetail(def re, def id, def headers) { Map params = new HashMap(); params.put("id", String.valueOf(id)) @@ -137,6 +161,17 @@ class SeaweedSchool { return JSONObject.parseObject(str, HouseDetail.class) } + private static School getNurserySchool(def re, def id, def headers) { + Map params = new HashMap(); + params.put("id", String.valueOf(id)) + params.put("type", "1") + params.put("page", "1") + params.put("perPage", "2") + params.put("source", "1") + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); + return JSONObject.parseObject(str, School.class) + } + private static School getPrimarySchool(def re, def id, def headers) { Map params = new HashMap(); params.put("id", String.valueOf(id)) @@ -144,7 +179,7 @@ class SeaweedSchool { params.put("page", "1") params.put("perPage", "2") params.put("source", "1") - def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); return JSONObject.parseObject(str, School.class) } @@ -155,7 +190,7 @@ class SeaweedSchool { params.put("page", "1") params.put("perPage", "4") params.put("source", "1") - def str = re.get("https://mapmapping.com/school-web/house/getPlotSchoolList", params, headers, "UTF-8"); + def str = re.get("https://mapmapping.com/school-web/school/getPlotSchoolList", params, headers, "UTF-8"); return JSONObject.parseObject(str, School.class) } } diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java index 1beb5153b..992727e92 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/seaweed/School.java @@ -1,6 +1,6 @@ package com.fangjia.script.statistics.seaweed; -import com.alibaba.fastjson.annotation.JSONField; +import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; import lombok.NoArgsConstructor; @@ -15,71 +15,52 @@ import java.util.List; public class School { - @JSONField(name = "code") + @JsonProperty("code") private String code; - @JSONField(name = "desc") + @JsonProperty("desc") private String desc; - @JSONField(name = "content") - private List content; - @JSONField(name = "timestamp") + @JsonProperty("content") + private ContentDTO content; + @JsonProperty("timestamp") private Long timestamp; @NoArgsConstructor @Data public static class ContentDTO { - @JSONField(name = "id") - private Integer id; - @JSONField(name = "name") - private String name; - @JSONField(name = "nature") - private Object nature; - @JSONField(name = "city") - private String city; - @JSONField(name = "area") - private String area; - @JSONField(name = "type") - private String type; - @JSONField(name = "quality") - private String quality; - @JSONField(name = "location") - private Object location; - @JSONField(name = "picture") - private String picture; - @JSONField(name = "phone") - private String phone; - @JSONField(name = "distanceNum") - private Object distanceNum; - @JSONField(name = "plotNum") - private String plotNum; - @JSONField(name = "sellNum") - private String sellNum; - @JSONField(name = "plotPrice") - private String plotPrice; - @JSONField(name = "address") - private Object address; - @JSONField(name = "lat") - private Object lat; - @JSONField(name = "lng") - private Object lng; - @JSONField(name = "scope") - private Object scope; - @JSONField(name = "score") - private Object score; - @JSONField(name = "scopeArr") - private Object scopeArr; - @JSONField(name = "scopeTag") - private Object scopeTag; - @JSONField(name = "tag") - private Object tag; - @JSONField(name = "shortName") - private Object shortName; - @JSONField(name = "view") - private Object view; - @JSONField(name = "construct") - private Object construct; - @JSONField(name = "echelon") - private Object echelon; - @JSONField(name = "town") - private Object town; + @JsonProperty("total") + private Integer total; + @JsonProperty("hasNextPage") + private Boolean hasNextPage; + @JsonProperty("list") + private List list; + + @NoArgsConstructor + @Data + public static class ListDTO { + @JsonProperty("id") + private Integer id; + @JsonProperty("name") + private String name; + @JsonProperty("area") + private String area; + @JsonProperty("shortName") + private Object shortName; + @JsonProperty("picture") + private String picture; + @JsonProperty("distance") + private String distance; + @JsonProperty("plotPrice") + private String plotPrice; + @JsonProperty("topInfo") + private Object topInfo; + @JsonProperty("view") + private String view; + @JsonProperty("town") + private String town; + @JsonProperty("address") + private String address; + @JsonProperty("tags") + private List tags; + } } } From d961902f86c17f7c8e17e2d091b7a7fd7726f378 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 5 Sep 2021 13:52:50 +0800 Subject: [PATCH 101/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=90=8D=E5=AD=97?= =?UTF-8?q?=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/SeaweedSchoolLatLng.groovy | 62 +++++++++---------- .../shenjian/seaweed/SeaweedSchool.groovy | 3 +- 2 files changed, 29 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy index d511bb745..93bf399b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -7,7 +7,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong @@ -19,7 +18,7 @@ class SeaweedSchoolLatLng { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) - static ExecutorService executorService = Executors.newFixedThreadPool(20) + static ExecutorService executorService = Executors.newFixedThreadPool(1) static void main(String[] args) { AtomicLong atomicLong = new AtomicLong(0); @@ -29,45 +28,40 @@ class SeaweedSchoolLatLng { @Override void run() { try { - BasicDBList primarySchools = obj.primarySchool - BasicDBList middleSchools = obj.middleSchool - if (primarySchools != null) { - for (BasicDBObject dbObject : primarySchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def address = obj.address + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } } - } - if (middleSchools != null) { - for (BasicDBObject dbObject : middleSchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } - collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } catch(Exception ex) { ex.printStackTrace() } finally { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index b988b6a9d..b3ab13fef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -48,7 +48,6 @@ class SeaweedSchool { final int perPage = 10 final int total = 4733; for (int page = 1; page < total / perPage; page++) { - Thread.sleep(1000) print("当前页:" + page) try { Map params = new HashMap(); @@ -71,7 +70,6 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) @@ -146,6 +144,7 @@ class SeaweedSchool { schoolObject.put("town", contentDTO.getTown()) schoolObject.put("distance", contentDTO.getDistance()) schoolObject.put("address", contentDTO.getAddress()) + schoolObject.put("name", contentDTO.getName()) schools.add(schoolObject) } return schools From 8d4c94431ce28ae8cabe56f72305a56ffbdcd866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Sun, 5 Sep 2021 13:52:50 +0800 Subject: [PATCH 102/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=90=8D=E5=AD=97?= =?UTF-8?q?=E4=B8=A2=E5=A4=B1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/SeaweedSchoolLatLng.groovy | 163 ++++++++++++++---- .../shenjian/seaweed/SeaweedSchool.groovy | 3 +- 2 files changed, 130 insertions(+), 36 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy index d511bb745..ad1676b7d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedSchoolLatLng.groovy @@ -7,7 +7,6 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong @@ -18,10 +17,17 @@ import java.util.concurrent.atomic.AtomicLong class SeaweedSchoolLatLng { static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "seaweed_school", "192.168.0.235", 27018) + static DBCollection school_collection = MongoUtils.getCollectionByName("fangjia_craw", "district_school", "192.168.0.235", 27018) static ExecutorService executorService = Executors.newFixedThreadPool(20) static void main(String[] args) { + //latLng() + updateAttribute() + executorService.shutdown() + } + + static void updateAttribute() { AtomicLong atomicLong = new AtomicLong(0); collection.find(new BasicDBObject()).batchSize(1000).each {objT -> final DBObject obj = objT @@ -29,45 +35,68 @@ class SeaweedSchoolLatLng { @Override void run() { try { - BasicDBList primarySchools = obj.primarySchool - BasicDBList middleSchools = obj.middleSchool - if (primarySchools != null) { - for (BasicDBObject dbObject : primarySchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def address = obj.address + boolean update = false + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'district_name': obj.name)) + def quality = '' + + if (primarySchools != null) { + if (schoolObj && schoolObj.attribute) { + quality = schoolObj.attribute + } + for (BasicDBObject dbObject : primarySchools) { + if (dbObject.name == '第六师范学校第二附属小学西校区' || String.valueOf(dbObject.name).startsWith('第六师范附属小学') + || dbObject.name == '上海市第六师范第二附属小学七号桥校区' || dbObject.name == '康城学校康佳校区' || dbObject.name == '三墩学校') { + dbObject.quality = '普通' + update = true + } else if (dbObject.name == '上海市实验学校东校(小学)' || dbObject.name == '浦东新区竹园小学沪东校区') { + dbObject.quality = '市重点' + update = true + } else if (dbObject.name == '浦东新区上南二村小学南校区' || dbObject.name == '浦东新区新时代小学大华校区' + || dbObject.name == '浦东新区明珠小学A区') { + dbObject.quality = '区重点' + update = true + } + else if (quality != '' && schoolObj.category == '小学') { + dbObject.quality = quality + update = true + } else { + update = updateDbObjectQuality(dbObject); } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } } - } - if (middleSchools != null) { - for (BasicDBObject dbObject : middleSchools) { - def name = dbObject.name - if (dbObject.lng == null) { - def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + name) - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) - dbObject.lng = lng - dbObject.lat = lat + def update2 = false + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + if (dbObject.name == '华东师范大学张江实验中学' || dbObject.name == '10上海市实验学校(初中)(全市招生)' || dbObject.name == '上海市实验学校附属光明学校总校区' + || dbObject.name == '上海市浦东新区进才实验中学南校' || String.valueOf(dbObject.name).startsWith('建平中学') || String.valueOf(dbObject.name).startsWith('建平实验中学') + || dbObject.name == '张江集团中学' || dbObject.name == '洋泾-菊园实验学校' || dbObject.name == '浦东外国语学校东校' || dbObject.name == '浦东新区建平中学南校' || String.valueOf(dbObject.name).startsWith('育人中学') + || dbObject.name == '致远中学' || String.valueOf(dbObject.name).startsWith('进才中学') || String.valueOf(dbObject.name).startsWith('进才实验中学')) { + dbObject.quality = '市重点' + update2 = true + } else if (dbObject.name == '华东师范大学附属东昌中学南校潍坊校区' || dbObject.name == '下沙学校鹤沙路校区' || dbObject.name == '傅雷中学关岳路校区' + || dbObject.name == '傅雷中学瑞和路校区' || dbObject.name == '华东师范大学附属东昌中学南校张江校区' || dbObject.name == '南汇第一中学卫星东路校区' + || dbObject.name == '康城学校(拯安校区)(初中)' || dbObject.name == '建平香梅中学' || dbObject.name == '沪新中学' || dbObject.name == '浦东模范中学' + || dbObject.name == '浦东模范中学东校' || dbObject.name == '浦东模范实验中学' || String.valueOf(dbObject.name).startsWith('澧溪中学') || + dbObject.name == '清流中学' || String.valueOf(dbObject.name).startsWith('王港中学') || dbObject.name == '罗山中学' + || dbObject.name == '金杨中学' || dbObject.name == '香山中学' || dbObject.name == '高行中学') { + dbObject.quality = '区重点' + update2 = true + } else if (quality != '' && schoolObj.category == '初中') { + dbObject.quality = quality + update2 = true + } else { + update2 = updateDbObjectQuality(dbObject) } } - if (dbObject.lng != null) { - String address = GaoDeUtils.getGaoDeAddress(dbObject.lng + "," + dbObject.lat) - dbObject.address = address - } + } + if (update || update2) { + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } } - collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) } catch(Exception ex) { ex.printStackTrace() } finally { @@ -81,6 +110,72 @@ class SeaweedSchoolLatLng { } }) }; - executorService.shutdown() + } + static boolean updateDbObjectQuality(BasicDBObject dbObject) { + if (dbObject.quality == '公办') { + DBObject schoolObj = school_collection.findOne(new BasicDBObject("city": '上海', 'school_name': dbObject.name)) + if (schoolObj && schoolObj.attribute) { + dbObject.quality = schoolObj.attribute + return true + } + } + return false + } + + static void latLng() { + AtomicLong atomicLong = new AtomicLong(0); + collection.find(new BasicDBObject()).batchSize(1000).each {objT -> + final DBObject obj = objT + executorService.submit(new Runnable() { + @Override + void run() { + try { + def address = obj.address + if (address.contains("浦东")) { + BasicDBList primarySchools = obj.primarySchool + BasicDBList middleSchools = obj.middleSchool + if (primarySchools != null) { + for (BasicDBObject dbObject : primarySchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + if (middleSchools != null) { + for (BasicDBObject dbObject : middleSchools) { + def schoolAddress = dbObject.address + if (dbObject.lng == null) { + def location = GaoDeUtils.parserByGaode("上海", "上海市浦东新区" + schoolAddress) + if (location && location.lng) { + def lng = Double.valueOf(location.lng) + def lat = Double.valueOf(location.lat) + dbObject.lng = lng + dbObject.lat = lat + } + } + } + } + collection.update(new BasicDBObject('url', obj.url), new BasicDBObject('$set', obj), false, true) + } + } catch(Exception ex) { + ex.printStackTrace() + } finally { + atomicLong.incrementAndGet() + synchronized (this) { + if (atomicLong.longValue() % 100 == 0) { + println(atomicLong.longValue()) + } + } + } + } + }) + }; } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy index b988b6a9d..b3ab13fef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/seaweed/SeaweedSchool.groovy @@ -48,7 +48,6 @@ class SeaweedSchool { final int perPage = 10 final int total = 4733; for (int page = 1; page < total / perPage; page++) { - Thread.sleep(1000) print("当前页:" + page) try { Map params = new HashMap(); @@ -71,7 +70,6 @@ class SeaweedSchool { List listDTOS = houseList.getContent().getList(); if (CollectionUtils.isNotEmpty(listDTOS)) { for (HouseList.ContentDTO.ListDTO listDTO : listDTOS) { - Thread.sleep(500) Integer id = listDTO.getId(); //# 房屋详情 HouseDetail houseDetail = getHouseDetail(re, id, headers) @@ -146,6 +144,7 @@ class SeaweedSchool { schoolObject.put("town", contentDTO.getTown()) schoolObject.put("distance", contentDTO.getDistance()) schoolObject.put("address", contentDTO.getAddress()) + schoolObject.put("name", contentDTO.getName()) schools.add(schoolObject) } return schools From 35ab32f65ebbb9b073035fbf0772e5d316be3e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 9 Sep 2021 15:44:24 +0800 Subject: [PATCH 103/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/dingding/DingTalkUtils.groovy | 2 +- .../temp/shenjian/poi/CommitteeLatLng.groovy | 17 +++++++++++------ .../script/temp/shenjian/util/GaoDeUtils.groovy | 12 +++++++----- 3 files changed, 19 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy index 209ab9fe2..795b1556a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/dingding/DingTalkUtils.groovy @@ -12,7 +12,7 @@ import com.fangjia.script.util.DateUtils class DingTalkUtils { static main(String[] args) { - Date now = DateUtils.parseFromTightByPattern("2021-07-01", "yyyy-MM-dd") ; + Date now = DateUtils.parseFromTightByPattern("2021-08-01", "yyyy-MM-dd") ; String startTime = DateUtils.addMonth(now, -1) + " 00:00:00" String monthStartTime = DateUtils.formatDateByPattern(now, "yyyy-MM-dd HH:mm:ss") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index 1f5f696cf..71d8c3bec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.poi +import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject @@ -18,7 +19,7 @@ import java.util.concurrent.Executors class CommitteeLatLng { static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) - static ExecutorService executorService = Executors.newFixedThreadPool(30) + static ExecutorService executorService = Executors.newFixedThreadPool(1) static void main(String[] args) { dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) @@ -37,12 +38,16 @@ class CommitteeLatLng { city = obj.province_name } def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; - def location = GaoDeUtils.parserByGaode(city, address); - if (location && location.lng) { - def lng = Double.valueOf(location.lng) - def lat = Double.valueOf(location.lat) + // 不准确 + // def location = GaoDeUtils.parserByGaode(city, address); + List amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "130106|130105|130104") + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) - .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "已获取经纬度"))) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) } else { villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 5334643d3..8496f3f01 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -128,9 +128,9 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); if (json == null) { - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/around", params); + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/around", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() @@ -169,11 +169,13 @@ class GaoDeUtils { /** * 查找关键字经纬度 */ - static List searchPoi(String city, String keywords){ + static List searchPoi(String city, String keywords, String region, String typecodes){ def params = [:] params.key = GAO_DE_AK[keyUsed.intValue()] params.keywords = keywords params.city = city + params.region = region + params.types = typecodes params.put("extensions", "all") params.put("citylimit","true") params.put("offset","20") @@ -184,9 +186,9 @@ class GaoDeUtils { while (result.size() < count) { params.page = "${page}".toString() page++ - JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + JSONObject json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); if (json == null) { - json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v3/place/text", params); + json = HttpRequestUtil.getJsonContent("http://restapi.amap.com/v5/place/text", params); } if (json && json.info && json.info == "OK") { count = json.getString("count").toInteger() From 613cd0db9b868bfb3f8473ebafad50d83fa1c620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Thu, 9 Sep 2021 16:16:38 +0800 Subject: [PATCH 104/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy | 6 ++++++ .../com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy | 5 ++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index 71d8c3bec..7edea330f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.temp.shenjian.poi +import com.fangjia.core.util.CosineSimilarAlgorithm import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.temp.shenjian.util.GaoDeUtils import com.fangjia.script.util.MongoUtils @@ -43,6 +44,11 @@ class CommitteeLatLng { List amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "130106|130105|130104") if (amapPoi && amapPoi.size() > 0) { def poi = amapPoi[0] + double similarity = CosineSimilarAlgorithm.getSimilarity(poi.name, obj.name) + if (similarity < 0.7) { + amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "") + } + poi = amapPoi[0] String[] locations = poi.location.split(",") def lng = Double.valueOf(locations[0]) def lat = Double.valueOf(locations[1]) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy index 8496f3f01..c75696cd6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/util/GaoDeUtils.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.statistics.poi.AmapPoi import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.HttpRequestUtil import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils +import org.apache.commons.lang.StringUtils import java.util.concurrent.atomic.AtomicInteger @@ -175,7 +176,9 @@ class GaoDeUtils { params.keywords = keywords params.city = city params.region = region - params.types = typecodes + if (StringUtils.isNotBlank(typecodes)) { + params.types = typecodes + } params.put("extensions", "all") params.put("citylimit","true") params.put("offset","20") From 69cb427f83fd7a6799ef2b49a04584b8f024a51e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 10 Sep 2021 11:42:36 +0800 Subject: [PATCH 105/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E7=BB=8F=E7=BA=AC?= =?UTF-8?q?=E5=BA=A6=E8=8E=B7=E5=8F=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/poi/CommitteeLatLng.groovy | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index 7edea330f..9a7ff1bb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -20,7 +20,7 @@ import java.util.concurrent.Executors class CommitteeLatLng { static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) - static ExecutorService executorService = Executors.newFixedThreadPool(1) + static ExecutorService executorService = Executors.newFixedThreadPool(20) static void main(String[] args) { dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0))) @@ -37,10 +37,10 @@ class CommitteeLatLng { def city = obj.city_name if (city == "市辖区") { city = obj.province_name + } else if (city == "省直辖县级行政区划") { + city = obj.area_name } def address = obj.province_name + city + obj.area_name + obj.street_name + obj.name; - // 不准确 - // def location = GaoDeUtils.parserByGaode(city, address); List amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "130106|130105|130104") if (amapPoi && amapPoi.size() > 0) { def poi = amapPoi[0] @@ -55,7 +55,18 @@ class CommitteeLatLng { villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) } else { - villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + amapPoi = GaoDeUtils.searchPoi(city, address, obj.area_name, "") + if (amapPoi && amapPoi.size() > 0) { + def poi = amapPoi[0] + String[] locations = poi.location.split(",") + def lng = Double.valueOf(locations[0]) + def lat = Double.valueOf(locations[1]) + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 0) + .append("location", new BasicDBObject("lat", lat).append("lon", lng)).append("flag_location", "关键字搜索获取经纬度"))) + + } else { + villagesConn.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', new BasicDBObject("status_location", 2).append("flag_location", "经纬度为空"))) + } } println("已处理小区:" + obj) } From d20b50d7617c939dd6cf8e310ee42dc9dc45d2a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Fri, 10 Sep 2021 14:45:14 +0800 Subject: [PATCH 106/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=92=89=E9=92=89?= =?UTF-8?q?=E5=AF=BC=E5=87=BA=E6=8A=A5=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/fangjia/script/statistics/ApiUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java index 7c87580de..746556da5 100644 --- a/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java +++ b/projects/fangjia-script/src/main/java/com/fangjia/script/statistics/ApiUtil.java @@ -347,7 +347,7 @@ public class ApiUtil { throw new RuntimeException("姣忎釜浼佷笟鍗曚釜鎺ュ彛姣忓ぉ鐨勬渶澶т笂闄愪负1000娆★紝鑻ヨ秴鍑轰細杩斿洖閿欒鐮�90009:getTaskDetail"); } } catch (Exception e) { - e.printStackTrace(); + //e.printStackTrace(); } return detail; } From 6b06ee211e4b1ba78cd51111123d5e9b272d2831 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Mon, 13 Sep 2021 09:22:23 +0800 Subject: [PATCH 107/212] =?UTF-8?q?excel=E4=B8=8D=E5=90=8C=E7=89=88?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 2e06fc2a1..3c1644892 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -366,12 +366,14 @@ org.apache.poi poi - 3.17 + + 3.10-FINAL org.apache.poi poi-ooxml - 3.17 + + 3.10-FINAL xml-apis @@ -476,21 +478,18 @@ aliyun-java-sdk-address-purification 1.0.0 + - com.alibaba - easyexcel - 2.2.8 - - - org.projectlombok - lombok - 1.18.20 - provided - - - org.msgpack - msgpack - 0.6.12 + com.eclipsesource.j2v8 + j2v8_win32_x86_64 + 4.6.0 + + + + elastic.co + https://artifacts.elastic.co/maven + + From f9a60cfa954b09fbf70a3c52db77895a02404d3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 14 Sep 2021 14:46:14 +0800 Subject: [PATCH 108/212] =?UTF-8?q?=E9=93=B6=E8=81=94=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- projects/fangjia-script/pom.xml | 59 ++++++++-------- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 34 ++++----- .../poi/YinlianLngLatAndAround.groovy | 69 +++++++++++++++++++ .../temp/shenjian/util/ExcelUtils.groovy | 9 +++ .../temp/shenjian/util/GaoDeUtils.groovy | 36 ++++++++++ 5 files changed, 163 insertions(+), 44 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index 3c1644892..bcebb2b2b 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -1,5 +1,5 @@ + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> 4.0.0 com.fangjia fangjia-script @@ -62,7 +62,7 @@ - org.eclipse.m2e @@ -141,11 +141,11 @@ activemq-pool 5.5.0
- - cn.emay - emayclient - 4.3 - + + cn.emay + emayclient + 4.3 + org.apache.activemq activemq-core @@ -366,20 +366,20 @@ org.apache.poi poi - - 3.10-FINAL + 3.17 + org.apache.poi poi-ooxml - - 3.10-FINAL - - - xml-apis - xml-apis - - + 3.17 + + + + xml-apis + xml-apis + + + + 3.10-FINAL org.apache.poi poi-ooxml - 3.17 - + + 3.10-FINAL xml-apis diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index a9ea036ef..29e3ac2a4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -22,7 +22,8 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + //def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"] def rows = [] def headers = [] for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy index 172aa4c7a..b59fe435b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommbuildtypeBatch.groovy @@ -41,7 +41,7 @@ class BigdataCommbuildtypeBatch extends BaseBatch{ rets.addAll(rows) println city +" "+rows.size() } - def headers = ["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] + def headers = []//["城市ID","小区ID"," 建筑类型","备注","创建时间","更新时间","来源公司","数据时间"] FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMBUILDTYPE_GJ.dat",headers, "utf-8", false); println "txt 生成" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 6e470ce2c..6241cc73b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]// CcbCommonUtils.citys; def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 6aabf6064..0564c1f41 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -267,18 +267,20 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) - -ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\分期数据删除_3144.xlsx",{d-> - if(d['A']){ - def udata = new BasicDBObject() - udata.put("name_cat", 9) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", new ObjectId(d['A'])) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - } -},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\第四批需要删除的数据.xlsx",{d-> +// if(d['A']){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", new ObjectId(d['A'])) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// index++ +// println index +// } +//},0) //ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> @@ -298,79 +300,79 @@ ExcelUtils.multiThreadingModify("D:\\ //},0) -//def index = 0 -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增别名数据_3788.xlsx",{d-> -// if(d['A']){ -// def dist = CommonDatautils.findDistrictById(d['A']) -// if(dist){ -// -// def alias = [] -// if(d['H'] && d['H']!=dist.name){ -// alias << d['H'] -// if(d['H'].contains("·")){ -// alias << d['H'].replace("·", "") -// } -// if(dist.alias){ -// for(def aliasName : dist.alias){ -// if(!alias.contains(aliasName)){ -// alias << aliasName -// } -// } -// } -// } -// -// def address = null -// if(d['I']){ -// address = AddressUtil.formatAddress(dist.city, d['I']) -// } -// -// Long lng = null -// Long lat = null -// Long lat2 = null -// Long lng2 = null -// if(d['J']){ -// lng = d['J'].toString().toDouble() * 10000000000l; -// lat = d['K'].toString().toDouble() * 10000000000l; -// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) -// if(json != null && json.containsKey("x")) { -// lat2 = json.get("y") * 10000000000l -// lng2 = json.get("x") * 10000000000l -// } -// } -// -// def region = null -// if(dist.region!=d['F']){ -// region = dist.region -// } -// -// def udata = new BasicDBObject() -// if(region){ -// udata.put("region", region) -// } -// -// if(address){ -// udata.put("address", address) -// } -// if(lat2){ -// udata.put("lng", lng) -// udata.put("lat", lat) -// udata.put("lng2", lng2) -// udata.put("lat2", lat2) -// } -// -// if(alias){ -// udata.put("alias", alias) -// } -// if(udata){ -// index++ -// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) -// println index+ " "+dist.city +" "+dist.name+" "+udata -// } -// -// } -// } -// -//},0) +def index = 0 +ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> + if(d['A']){ + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + + def alias = [] + if(d['H'] && d['H']!=dist.name){ + alias << d['H'] + if(d['H'].contains("·")){ + alias << d['H'].replace("·", "") + } + if(dist.alias){ + for(def aliasName : dist.alias){ + if(!alias.contains(aliasName)){ + alias << aliasName + } + } + } + } + + def address = null + if(d['I']){ + address = AddressUtil.formatAddress(dist.city, d['I']) + } + + Long lng = null + Long lat = null + Long lat2 = null + Long lng2 = null + if(d['J']){ + lng = d['J'].toString().toDouble() * 10000000000l; + lat = d['K'].toString().toDouble() * 10000000000l; + def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) + if(json != null && json.containsKey("x")) { + lat2 = json.get("y") * 10000000000l + lng2 = json.get("x") * 10000000000l + } + } + + def region = null + if(dist.region!=d['F']){ + region = dist.region + } + + def udata = new BasicDBObject() + if(region){ + udata.put("region", region) + } + + if(address){ + udata.put("address", address) + } + if(lat2){ + udata.put("lng", lng) + udata.put("lat", lat) + udata.put("lng2", lng2) + udata.put("lat2", lat2) + } + + if(alias){ + udata.put("alias", alias) + } + if(udata){ + index++ + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + println index+ " "+dist.city +" "+dist.name+" "+udata + } + + } + } + +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy new file mode 100644 index 000000000..6ee9f1f56 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/CcbDistrictToDB.groovy @@ -0,0 +1,137 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.mongodb.BasicDBObject + +class CcbDistrictToDB { + + static main(args) { + def task = new CcbDistrictToDB(); + task.run() + } + + def run(){ + def files = ["北京融合匹配_1215.xlsx","广州融合匹配_1215.xlsx","杭州融合匹配_1216.xlsx","合肥融合匹配_1216.xlsx","深圳融合匹配_1215.xlsx","石家庄融合匹配_1216.xlsx","苏州融合匹配_1216.xlsx","武汉融合匹配_1216.xlsx","西安融合匹配_1216.xlsx","重庆融合匹配_1215.xlsx"] + def index = 0 + for(def f : files){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\融合\\10个城市新增小区\\" + f, { d -> +// if(d['I']=="棠德花苑新社区"){ + def city = d['A'] + def region = d['C'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + + def name = "" + def alias = [] + if(d['I']){ + name = nameFileter(d['I']) + if(name != d['I']){ + alias << d['I'] + } + } + if(name){ + if(d['D']!=name && !alias.contains(d['D'])){ + alias << d['D'] + } + }else{ + name = d['D'] + } + + def formatAddress = null + if(d['J']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['J']) + } + if(!formatAddress && d['E']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['E']) + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['K']){ + glat = d['K'].toString().toDouble() * 10000000000L + glng = d['L'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['K'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + }else{ + if(d['F']){ + def loactionMap = GeoUtils.gps84ToGcj02(d['F'], d['G']) + def lat = null + def lng = null + if(loactionMap){ + lng = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) + lat = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) + } + def LatLon = GeoHelper.GetLatLon(lat, lng, 0.01, 180); + def distance = GeoHelper.GetDistance(lat, lng, LatLon.Lat, LatLon.Lon) + + glat = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lat, 6) * 10000000000L + glng = com.fangjia.core.util.DataUtil.roundDouble(LatLon.Lon, 6) * 10000000000L + def baidu = BaiDuApiUtils.geoconv(LatLon.Lon, LatLon.Lat, 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + } + + if(name){ + def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", d['A']) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + if(alias){ + data.put("alias",alias) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + } + } + } +// } + },0); + } + } + + def nameFileter(def name){ + name = name.replace("·","").replace(".","").replace("|","") + name = name.replace("1期","一期").replace("2期","二期").replace("3期","三期").replace("4期","四期").replace("5期","五期").replace("6期","6期") + name = name.replace("1区","一区").replace("2区","二区").replace("3区","三区").replace("4区","四区").replace("5区","五区").replace("6区","六区") + return name; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy new file mode 100644 index 000000000..88156feec --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy @@ -0,0 +1,150 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.apache.poi.ss.util.RegionUtil +import org.bson.types.ObjectId +import com.fangjia.core.util.BaiDuApiUtils + +class DistrictMarge { + static main(args) { + def task = new DistrictMarge() + task.run() + } + + def run(){ + def rets = getDatas() + def deleteIndex = 0 + def updateIndex = 0 + rets.each{amap_id, valus-> + def updateObj = null + def addAlias = [] + String formatAddress = null + Long lat2 = null + Long lng2 = null + Long lat = null + Long lng = null + def amapName = null + for(def i=0;i + Collections.sort(v,new Comparator(){ + public int compare(def o1, def o2) { + + if(o1['D'].toString().length() > o2['D'].toString().length()){ + return 1; + } + if(o1['D'].toString().length() == o2['D'].toString().length()){ + return 0; + } + return -1; + } + }) + } + return map; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy new file mode 100644 index 000000000..e5ec16dfb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -0,0 +1,333 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.mongodb.BasicDBObject +import com.mongodb.DBObject +import org.apache.poi.ss.util.RegionUtil +import org.elasticsearch.index.query.BoolQueryBuilder +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders + +class MatchMapData { + + def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new MatchMapData() +// task.run() +// task.run1() +// task.run2() + task.run3() +// task.exportHouse() + + } + + def run3(){ + + def files = ["合肥融合匹配_1215.xlsx"] + + for(def fn : files){ + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合匹配房源\\"+fn,{d-> + def city = d['A'] + def region = d['D'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def count = findHouses2(city, newRegion, "sell", 20160101, d['E']); + if(count){ + d['O'] = "有" + }else{ + count = findThirdParty(city, newRegion, d['E']); + if(count){ + d['O'] = "有" + }else{ + count = findThirdParty(city, region, d['E']); + if(count){ + d['O'] = "有" + } + } + } + } + },0) + } + } + + def run2(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> + def dist = CommonDatautils.findDistrictById(d['A']) + println dist.name + },0) + } + + def run1(){ + //"合肥","北京","广州","杭州","上海","深圳","石家庄","天津","武汉", + def files = ["苏州"] + for(def f : files){ + def city = f + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d-> + if(d['E']){ + def region = d['E'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def rMap = match(city, newRegion, d) + if(rMap){ + def priceObj = CommonDatautils.findDistrictPrice(rMap.city, rMap.region, rMap.name) + d['M'] = rMap.region + d['N'] = rMap.name + d['O'] = rMap.address + if(priceObj){ + d['P'] = priceObj.avg_price + } + + def dist = CommonDatautils.findDistrictById(rMap.id) + if(dist){ + d['Q'] = dist.lng/10000000000.0 + d['R'] = dist.lat/10000000000.0 + def count = findHouses(dist.city, dist.region, "rent", 20210801, 20210901, dist.name, dist.alias); + if(count){ + d['V'] = "有" + } + } + + if(d['I']){ + def loactionMap = GeoUtils.gps84ToGcj02(d['H'], d['I']) + if(loactionMap){ + d['S'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lng,6) + d['T'] = com.fangjia.core.util.DataUtil.roundDouble(loactionMap.lat, 6) + } + + def distance = GeoHelper.GetDistance(d['R'], d['Q'], d['T'], d['S']) + d['U'] = distance + } + + + } + } + } + + },0) + } + } + + def match(city, region, d){ + def rMap = SeaweedDataMatchProcess.doMatch(city, region, d['F'], null); + if(!rMap && d['G']){ + def addrFlag = d['G'].toString().matches('^[\u4e00-\u9fa5]+?(路|大道|街).+[弄号]$') + if(addrFlag){ + rMap = SeaweedDataMatchProcess.doMatch(city, region, d['G'], null); + } + } + return rMap; + } + + def run(){ + + def files = ["北京","广州","杭州","合肥","上海","深圳","石家庄","天津","武汉","西安","重庆"] + for(def f : files){ + def city = f + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\"+f+".xlsx",{d-> + if(d['E']){ + def region = d['E'] + def newRegion = RegionData.formatRegion(city, region) + + if(newRegion){ + def dist = CommonDatautils.findDistrict(city, newRegion, d['F'],"district") + if(dist){ + d['K']="完全一致" + d['L'] = dist.name_cat + }else{ + dist = CommonDatautils.findSeaweedByMappingAlias(city, newRegion, d['F'],"district") + if(dist){ + d['K']="映射别名" + d['L'] = dist.name_cat + }else{ + dist = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district") + if(dist){ + d['K']="别名" + } + } + } + }else{ + d['M'] = "区域无法标准化" + } + } + + },0) + } + + } + + + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + return houses.size(); + } + + + + def findHouses2(def city, def region, def cat, def startDate, def name){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).includeLower(true); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(date_range).must(area_range) + + def alias = [] + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + if(name.contains("号小区")){ + def newName = name.replace("号小区","号") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + + println "[房源] "+city +" "+region+" "+ name +" "+houses.size() + return houses.size(); + } + + + def findThirdParty(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city",city) + q.put("region",region) + q.put("name",name) + def count = thirdPartyColl.count(q) + println "[第三方] "+city +" "+region+" "+ name +" "+count + return count + } + + def exportHouse(){ + def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"] + for(def f : files) { + def city = f + + def rets = [] + rets << ["建行小区id", "房价网小区id","城市","区域","楼盘名称","单价","月租金","面积","室","厅","卫","出租方式","朝向","楼屋","装修","发布时间"] + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\" + f + ".xlsx", { d -> + def flag = d['K'] + if(flag && flag =="完全一致"){ + def obj = CommonDatautils.findDistrict(city, d['M'], d['N'], "district") + if(obj){ + def houses = findHouses3(d['D'], obj._id.toString(), city, d['M'], "rent", 20210801, 20210901, d['N'], null); + if(houses){ + rets.addAll(houses) + } + } + + } + },0) + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房\\"+city+"_rent.xlsx", rets, "数据") + } + } + + + + def findHouses3(def commid, def fjwid, def city, def region, def cat, def startDate, def endDate, def name, def alias){ + + QueryBuilder date_range = QueryBuilders.rangeQuery("p_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) + .must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range) + + if(!alias){ + alias = [] + } + if(!alias.contains(name)){ + alias.add(name) + } + + + if(name.contains("弄小区")){ + def newName = name.replace("弄小区","弄") + if(!alias.contains(newName)){ + alias << newName + } + + } + + if(alias && alias.size()>0){ + BoolQueryBuilder inBuilder = QueryBuilders.boolQuery(); + for(def s : alias){ + inBuilder.should(QueryBuilders.matchQuery("district_name", s)); + } + queryBuilder.must(inBuilder) + } + + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + def rets = [] + for(def r : houses){ + rets << [commid, fjwid, r.city,r.region,r.district_name, r.avg_price, r.total_price, r.area,r.room, r.hall,r.toilet,r.rent_type, r.direction,r.floor_info,r.fitment, r.p_date ] + } + + if(rets){ + println city+" "+ name +" "+rets.size() + } + return rets; + } + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 87e8f37fa..599442639 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]//CcbCommonUtils.citys; def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 40b3605c1..596a0c6a6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -24,7 +24,8 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { - def citys = CcbCommonUtils.citys; +// def citys = CcbCommonUtils.citys; + def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // exportAllBigdataDealFJW() @@ -34,17 +35,17 @@ class ExportDistrictRongHe extends BaseBatch{ //// println city +" "+cityCode //// } exportBigdataCommunityGJRH(cityCode) -// exportBigdataBuildingFJW(cityCode) -// exportBigdataCommfeeGj(cityCode) + exportBigdataBuildingFJW(cityCode) + exportBigdataCommfeeGj(cityCode) // -// exportBigdataDealFJW(cityCode) -// exportBigdataCommbuildstruGJ(cityCode) -// exportBigdataCommhousetypeGJ(cityCode) -// exportBigdataCommbuildyearGj(cityCode) -// exportBigdataCommhouseuseGJ(cityCode) -// exportBigdataPositionGJ(cityCode) -// exportBigdataDistrictFJW(cityCode) -// exportBigdataAliasFJW(cityCode) + exportBigdataDealFJW(cityCode) + exportBigdataCommbuildstruGJ(cityCode) + exportBigdataCommhousetypeGJ(cityCode) + exportBigdataCommbuildyearGj(cityCode) + exportBigdataCommhouseuseGJ(cityCode) + exportBigdataPositionGJ(cityCode) + exportBigdataDistrictFJW(cityCode) + exportBigdataAliasFJW(cityCode) // // exportBigdataBuildprojFjw(cityCode) // exportBigdataCommlayoutpicFjw(cityCode) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 87c6ba7ed..1549f70c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -204,8 +204,6 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "安居小区"); - DSCursor cursor = getDataStoreConnection().find(q); @@ -1171,6 +1169,37 @@ class CommonDatautils { return ret; } + + static def findSeaweedByMappingAlias(def city, def region, def name, def cat){ + DBObject q = new BasicDBObject(); + if(region){ + q.put('region', region) + } + + q.put('city',city) + q.put('cat', cat) + q.put('status',0) + + BasicDBObject query = new BasicDBObject(); + query.putAt(QueryOperators.IN, [name]); + q.put("mapping_alias", query); + + def obj = SEAWEED_COLL.findOne(q); + return obj; + } + + static def findAllSeaweed(def cat , def city){ + DBObject q = new BasicDBObject(); + q.put('city',city) + q.put('cat', cat) + q.put('status',0) + def dbcursor = SEAWEED_COLL.find(q).batchSize(5000) + def rets = [] + for(def d : dbcursor){ + rets << d + } + return rets; + } static def findSeaweedByAddress(def city, def region, def address, def cat){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/SynchSeaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/SynchSeaweed.groovy new file mode 100644 index 000000000..62e182a5b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/SynchSeaweed.groovy @@ -0,0 +1,45 @@ +package com.fangjia.script.gj + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection + +class SynchSeaweed { + + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + public static final def offlineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.136:28017"; + + public static final DBCollection SEAWEED_ONLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed", mongoDbUrl) + + public static final DBCollection SEAWEED_OFFLINE_COLL = MongoUtils.getCollectionByName2("fangjia", "seaweed_1215", offlineUrl) + + static main(args) { + def task = new SynchSeaweed(); + task.run() + } + + def run(){ + def citys = CommonDatautils.findAllCity(); + for(def city : citys){ + def datas = CommonDatautils.findAllSeaweed("district", city) + saveDatas(datas); + } + } + + def saveDatas(def datas){ + def index = 0 + def ret = [] + for(def d : datas){ + index++; + ret << d + if(ret.size()%10000==0){ + println "第 "+(index/10000)+" 个"+ret.size() + SEAWEED_OFFLINE_COLL.insert(ret); + ret = []; + } + } + if(ret){ + SEAWEED_OFFLINE_COLL.insert(ret) + } + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy index 9219fc438..319e44ef4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/SeaweedDataMatchProcess.groovy @@ -29,6 +29,7 @@ class SeaweedDataMatchProcess { rMap.lat = obj.lat/10000000000.0 rMap.lng = obj.lng/10000000000.0 rMap.id = obj._id.toString() + rMap.address = obj.address } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy index 2c4adeffd..8d353b9b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/es/EsHouseMatchDistrict.groovy @@ -25,7 +25,7 @@ import java.util.zip.GZIPInputStream class EsHouseMatchDistrict { def targetUrl = "http://192.168.0.206:9200/seawater-2021/_doc" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) static main(args) { def task = new EsHouseMatchDistrict() @@ -33,9 +33,9 @@ class EsHouseMatchDistrict { } def houseRun(){ - def citys = ["苏州","杭州"]//CommonUtils.findAllCity(); + def citys = ["杭州"]//CommonUtils.findAllCity(); for(def city : citys){ - def endDate = DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()) + def endDate = DateUtils.parseFromTightByPattern("2021-06-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()) def startDate = DateUtils.parseFromTightByPattern("2021-01-01","yyyy-MM-dd")//DateUtils.truncateDay(new Date()-1) def houses = findHouseCounts(city, startDate, endDate); match(city, houses); @@ -59,8 +59,9 @@ class EsHouseMatchDistrict { if(rMap){ successCount.addAndGet(1) updateHouseMatchDistrictId(t.id, rMap.id) + println successCount.get()+" "+t.id+" "+ rMap } - println count.get()+" "+t.id+" "+ rMap + } }catch(Exception e){ e.printStackTrace() @@ -82,7 +83,7 @@ class EsHouseMatchDistrict { } def findHouseCounts(def city, def startDate, def endDate){ - QueryBuilder date_range = QueryBuilders.rangeQuery("m_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); + QueryBuilder date_range = QueryBuilders.rangeQuery("publish_date").from(startDate).to(endDate).includeLower(true).includeUpper(false); QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)) .must(QueryBuilders.termQuery("cat", "sell")) @@ -94,13 +95,14 @@ class EsHouseMatchDistrict { // queryBuilder.must(inBuilder) def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "seawater-20*", queryBuilder); + println houses.size() def rets = [] - for(def r : rets){ + for(def r : houses){ if(!r.match_seaweed_id){ rets << r } } - return houses + return rets } @@ -122,7 +124,8 @@ class EsHouseMatchDistrict { } """ HttpRequest request = new HttpRequest() - request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") + println s } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy index c14e1d3ba..517b6b634 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/linxsh/util/AddressUtil.groovy @@ -120,6 +120,9 @@ class AddressUtil { if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷).+[弄号]$')) { return formatAddress } + if(StringUtils.isNotBlank(formatAddress) && formatAddress.matches('^[\u4e00-\u9fa5]+?(路|大道|街|巷)')) { + return formatAddress + } return null } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 751d405bb..6566d5a16 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,7 +40,7 @@ class DataStat { def run(){ - def source = "我爱我家"// + def source = "城市房产"// def citys = findCrawlCity(source, 202110); for(def city : citys){ def count1 = findCount(source, city, 202110); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 83adfa245..3317161c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -37,8 +37,9 @@ class Esf58Crawl extends BaseBatch{ def static final distMap = [:] def static useDistKeys = [] static def totalCount = new AtomicLong() - //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", - def cityList = [ "上海":"sh|2","北京":"bj|1","天津":"tj|18","重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79", + //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", + //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4", + def cityList = ["东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", "青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", @@ -82,7 +83,7 @@ class Esf58Crawl extends BaseBatch{ "哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", - "肇州":"zhaozhou|382", "武汉":"wh|158","襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 94f0e3a4c..c106ca013 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,7 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["cityhouse.cn":"城市房产"] + def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy index ef14c99b0..0b5a49b5d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/ThirdDataToGatherBatch.groovy @@ -40,7 +40,6 @@ class ThirdDataToGatherBatch extends BaseBatch{ static main(args) { runBatch(new ThirdDataToGatherBatch(), args) } - @Override void process(Object args) { println SHORT_DATE From baf4c7c3610dd97d7036ebb2e8abc1b76ba9ffc8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 20 Dec 2021 18:00:21 +0800 Subject: [PATCH 137/212] update --- .../script/ccbevaluation/BigdataCommunityBatch.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy index 6241cc73b..c2f8fe431 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataCommunityBatch.groovy @@ -35,7 +35,7 @@ class BigdataCommunityBatch extends BaseBatch{ @Override public void process(Object args) { - def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]// CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def headers = []//["城市ID","行政区ID","小区ID","行政区名称","小区名称","详细地址","纬度","经度","占地总面积","建筑总面积","绿化率","容积率","车位数量","建筑年代","房屋用途", @@ -44,6 +44,9 @@ class BigdataCommunityBatch extends BaseBatch{ def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() for(def city : citys){ +// if(["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"].contains(city)){ +// continue +// } def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def dists = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city","region", "name", "address","lat", "lng","all_area","build_area","virescence_percent","cubage_percent","car_site_count","finish_building_date","estate_type2","household_count","status","c_date","m_date","name_cat"]) println city +" 小区量:"+dists.size() From 516ea36f15af7487643a3902f3087d7230504087 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 15:52:16 +0800 Subject: [PATCH 138/212] update --- .../ccbevaluation/BigdataDealBatch.groovy | 18 +- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 2 +- .../DealAndThirdPriceDataProcess.groovy | 109 ++++++++++- .../ccbevaluation/ExportOracleData.groovy | 148 +++++++++++++-- .../fangjia/script/ccbevaluation/Test.groovy | 175 +++++++++--------- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +- .../script/ccbevaluation/TradeStatLog.groovy | 14 +- .../ccbevaluation/data/MatchMapData.groovy | 38 +++- .../ronghe/BigdataCommunityGJRHBatch.groovy | 4 +- .../ronghe/ExportDistrictRongHe.groovy | 23 +-- .../script/commData/CommonDatautils.groovy | 16 ++ .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 4 +- 14 files changed, 406 insertions(+), 157 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 03bc56f0a..39b653a2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -32,7 +32,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class BigdataDealBatch extends BaseBatch{ - def curDate = 20211001//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); + def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMMdd"), -1, "yyyyMMdd")); public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def sourceMap = ["贝壳找房":"ke.com","我爱我家":"5i5j.com","链家在线":"lianjia.com","房天下":"fang.com","中原地产":"centanet.com","福州不动产":"","搜房":"fang.com"] @@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{ //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; -// for(def city : citys){ -// addDealPriceMatch(city); -// } + for(def city : citys){ + addDealPriceMatch(city); + } println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] @@ -443,15 +443,9 @@ class BigdataDealBatch extends BaseBatch{ // def citys = ["北京","广州","深圳","重庆","郑州","西安","成都","东莞","佛山","杭州","合肥","南京","长春","扬州","无锡","太原","台州","泉州","宁波","南通","南昌","贵阳","哈尔滨","惠州","济南","金华","昆明","遵义","镇江","肇庆","宜昌","芜湖","泰州","汕头","清远","秦皇岛","马鞍山","洛阳","连云港","九江","淮安","呼和浩特","沧州","保定","安庆"] - def startDate = null; -// if(citys.contains(city)){ - startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) -// }else{ -// startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern("20210601", "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) -// } + def startDate = DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd")) def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern(curDate.toString(), "yyyyMMdd"))//DateUtils.getFirstDayOfMonth(new Date()) -// def endDate = DateUtils.getLastDayOfMonth(DateUtils.parseFromTightByPattern("20210801", "yyyyMMdd")) for(def i=0;i m.deal_price){ + newPrice = m.deal_price*precent + m.deal_price + }else if(m.unit_price < m.deal_price){ + newPrice = m.deal_price - (m.deal_price*precent) + } + if(newPrice){ + updateDealPrice(m, newPrice.toInteger()) + } + } + } + +// if(d['M']){ +// def rets = findThirdByPe(d['A'], yearMonth, d['M']) +// for(def r : rets){ +// deletThirdPrice(r, thirdMonth) +// } +// } + } + } void process() { @@ -96,13 +138,18 @@ class DealPriceDataProcess { def deleteDealPrice(r){ def sql = "delete from bigdata_deal_gj where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} " + "and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')" - sqlInstance.executeUpdate(sql) + sqlInstance236.executeUpdate(sql) + } + + def deletThirdPrice(r, thirdMonth){ + def sql = "delete from bigdata_third_party_price_gj where price_date=${thirdMonth} and commid='${r.commid}' and community_price=${r.target_avg_price}" + sqlInstance236.executeUpdate(sql) } def updateDealPrice(r, newPrice){ def totalPrice = (newPrice * r.buildarea).toInteger() def sql = "update bigdata_deal_gj set DEALUNITPRICE=${newPrice},DEALTOTALPRICE=${totalPrice} where commid='${r.commid}' and buildarea=${r.buildarea} and DEALUNITPRICE=${r.deal_price} and dealdate=to_date('${r.dealdate}','yyyy-mm-dd')" - sqlInstance.executeUpdate(sql) + sqlInstance236.executeUpdate(sql) } def findLtePPeCount(city, year_month){ @@ -114,6 +161,39 @@ class DealPriceDataProcess { return count } + def findDealByPe(city, yearMonth, peVal){ + def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.cityname = r.cityname + map.cityid=r.cityid + map.distid = r.distid + map.commid = r.commid + map.commname = r.commname + map.deal_price = r.target_avg_price + map.unit_price = r.unit_price + map.buildarea = r.buildarea + map.ppe = r.pe + map.dealdate = r.dealdate + rets << map + }) + return rets + } + + def findThirdByPe(city, yearMonth, peVal){ + def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.year_month = r.year_month + map.commid = r.commid + map.target_avg_price = r.target_avg_price + rets << map + }) + return rets + } + def findAllGtePPeDatas(city, yearMonth){ def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>15 order by pe asc"; @@ -147,6 +227,27 @@ class DealPriceDataProcess { }); return map } + + + def findCityByPpeRange(def cityname, def yearMonth, ppeStart, ppeEnd){ + def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where cityname='${cityname}' and year_month=${yearMonth} and data_type='成交' and pe>${ppeStart} and pe<${ppeEnd}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.cityname = r.cityname + map.cityid=r.cityid + map.distid = r.distid + map.commid = r.commid + map.commname = r.commname + map.deal_price = r.target_avg_price + map.unit_price = r.unit_price + map.ppe = r.pe + map.buildarea = r.buildarea + map.dealdate = r.dealdate + rets << map + }); + return rets + } def getCityPPeIndex(def filePath){ def datas = ExcelUtils.read(filePath,0,1) def map = [:] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index d6ba74db8..4e28e77d6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -11,7 +11,7 @@ import java.sql.ResultSet class ExportOracleData { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -24,30 +24,36 @@ class ExportOracleData { } def run() { + +// exportGuidedPrice() +// exportCommunityGuidedPrice() +// +// //exportThirdPartyPrice()//废弃掉,老表 + exportThirdPartyPrice2(202111) +// +// +// +// exportBigdataForsale("2021-11-19") +// // exportEvaluationRateData() // exportBigdataPoiData() // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() -// exportBigdataDeal("2021-10-01") + exportBigdataDeal("2021-11-01") -// exportBigdataForsale("2021-10-19") - exportGuidedPrice() - exportCommunityGuidedPrice() - -// exportThirdPartyPrice()//废弃掉,老表 -// exportThirdPartyPrice2() - - - exportAvmEvaluationGJNB(202111) - exportAVMEvaluationItemGJNB(202111) - exportCommunityBasePrice(202111) +// exportAvmEvaluationGJNB(202111) +// exportAVMEvaluationItemGJNB(202111) +// exportCommunityBasePrice(202111) // exportCityPrice(202111) // exportRegionPrice(202111) // exportCommunityBasePriceGj(202111) + +// exportCommunityPdescribeGj(202112) +// exportCommCaseDetailGj(202112) // exportLastFile() } @@ -422,8 +428,8 @@ class ExportOracleData { } - def exportThirdPartyPrice2() { - def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ"; + def exportThirdPartyPrice2(price_date) { + def sql = "select ID, CITYID, DISTID, COMMID, COMMNAME, DATA_SOURCE_ID, DATA_SOURCE, PRICE_DATE, COMMUNITY_PRICE, COMMUNITY_MAX_PRICE, COMMUNITY_MIN_PRICE, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME from BIGDATA_THIRD_PARTY_PRICE_GJ where price_date=${price_date} "; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -444,8 +450,8 @@ class ExportOracleData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { -// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" - fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" +// fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] //["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","均价对应的月份","诸葛找房均价","安居客均价","房天下均价","链家均价","城市房产均价","麦田均价","贝壳均价","淘房365","中原均价","兔博士均价","Q房均价","房探均价","房多多均价","我爱我家均价","创建时间","更新时间","来源公司"] @@ -858,6 +864,114 @@ class ExportOracleData { println "DISTRICT_PRICE_GJ 上传Ftp完成......" } + def exportCommunityPdescribeGj(month){ + + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys) { + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} and cityid='${cityCode}'"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println city + " "+allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(month){ + + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'";//price_date=${month} and + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + def exportLastFile(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def dataCodeFileName = "/ccb/${curDate}.ok" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0564c1f41..9450808e8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -30,6 +30,9 @@ import java.util.regex.*; def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def sqlInstance94 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -221,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -//def fileName = "D:\\数据\\建行\\建行成交\\成交数据_1025 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance, "2021-10-01", "2021-11-01") -//def testMap = findCaluateCityPPE(sqlInstance94, 202111) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['N'] = oldcount -// def casecount = testMap.get(d['C']) -// d['O'] = casecount -// println d -// } -//},0) +def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx" +def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01") +def testMap = findCaluateCityPPE(sqlInstance236, 202112) +ExcelUtils.modify(fileName,{d-> + if(d['C']){ + def oldcount = dealMap.get(d['C']) + d['P'] = oldcount + def casecount = testMap.get(d['C']) + d['Q'] = casecount + println d + } +},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -300,79 +303,79 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //},0) -def index = 0 -ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> - if(d['A']){ - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - - def alias = [] - if(d['H'] && d['H']!=dist.name){ - alias << d['H'] - if(d['H'].contains("·")){ - alias << d['H'].replace("·", "") - } - if(dist.alias){ - for(def aliasName : dist.alias){ - if(!alias.contains(aliasName)){ - alias << aliasName - } - } - } - } - - def address = null - if(d['I']){ - address = AddressUtil.formatAddress(dist.city, d['I']) - } - - Long lng = null - Long lat = null - Long lat2 = null - Long lng2 = null - if(d['J']){ - lng = d['J'].toString().toDouble() * 10000000000l; - lat = d['K'].toString().toDouble() * 10000000000l; - def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) - if(json != null && json.containsKey("x")) { - lat2 = json.get("y") * 10000000000l - lng2 = json.get("x") * 10000000000l - } - } - - def region = null - if(dist.region!=d['F']){ - region = dist.region - } - - def udata = new BasicDBObject() - if(region){ - udata.put("region", region) - } - - if(address){ - udata.put("address", address) - } - if(lat2){ - udata.put("lng", lng) - udata.put("lat", lat) - udata.put("lng2", lng2) - udata.put("lat2", lat2) - } - - if(alias){ - udata.put("alias", alias) - } - if(udata){ - index++ - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) - println index+ " "+dist.city +" "+dist.name+" "+udata - } - - } - } - -},0) +//def index = 0 +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕).xlsx",{d-> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// +// def alias = [] +// if(d['H'] && d['H']!=dist.name){ +// alias << d['H'] +// if(d['H'].contains("·")){ +// alias << d['H'].replace("·", "") +// } +// if(dist.alias){ +// for(def aliasName : dist.alias){ +// if(!alias.contains(aliasName)){ +// alias << aliasName +// } +// } +// } +// } +// +// def address = null +// if(d['I']){ +// address = AddressUtil.formatAddress(dist.city, d['I']) +// } +// +// Long lng = null +// Long lat = null +// Long lat2 = null +// Long lng2 = null +// if(d['J']){ +// lng = d['J'].toString().toDouble() * 10000000000l; +// lat = d['K'].toString().toDouble() * 10000000000l; +// def json = BaiDuApiUtils.geoconv(d['J'].toString().toDouble(), d['K'].toString().toDouble(), 3) +// if(json != null && json.containsKey("x")) { +// lat2 = json.get("y") * 10000000000l +// lng2 = json.get("x") * 10000000000l +// } +// } +// +// def region = null +// if(dist.region!=d['F']){ +// region = dist.region +// } +// +// def udata = new BasicDBObject() +// if(region){ +// udata.put("region", region) +// } +// +// if(address){ +// udata.put("address", address) +// } +// if(lat2){ +// udata.put("lng", lng) +// udata.put("lat", lat) +// udata.put("lng2", lng2) +// udata.put("lat2", lat2) +// } +// +// if(alias){ +// udata.put("alias", alias) +// } +// if(udata){ +// index++ +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// println index+ " "+dist.city +" "+dist.name+" "+udata +// } +// +// } +// } +// +//},0) @@ -440,7 +443,7 @@ ExcelUtils.multiThreadingModify("D:\\ def findDealCount(sqlInstance, def startDate, def endDate){ - def sql = "select cityname,count(*) as count from bigdata_deal_gj_test where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate + def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365) + def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear)) + def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend) + + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d-> if(d['E']){ def region = d['E'] def newRegion = RegionData.formatRegion(city, region) @@ -148,12 +152,34 @@ class MatchMapData { d['K']="映射别名" d['L'] = dist.name_cat }else{ - dist = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district") - if(dist){ + def rets = CommonDatautils.findDistrictByAlias(city, newRegion, d['F'],"district") + if(rets){ d['K']="别名" } + if(rets){ + dist = rets[0] + } } } + + if(dist){ + def nearDbo = CommonDatautils.findDistrictPrice(dist.city, dist.region, dist.name) + + def lastMonth = DateUtils.parseFromTightYYYYMMDD("2021-11-30") + def sqmetre = CommonDatautils.findSqmetreRent(dist.city, dist.region, dist.name, lastMonth) + if(nearDbo && districtPriceMap){ + def priceObj = districtPriceMap.get(dist.region+"#"+dist.name) + if(priceObj){ + def markup = (nearDbo.avg_price - priceObj.avg_price)*100/priceObj.avg_price + d['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, markup) + } + } + if(nearDbo && sqmetre){ + def prc = nearDbo.avg_price/sqmetre.avg_price + d['N'] = prc.toInteger() + } + d['O'] = dist.estate_type2 + } }else{ d['M'] = "区域无法标准化" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 599442639..b25c8bc10 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -32,7 +32,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") public static def batchsize = 1000 @@ -42,7 +42,7 @@ class BigdataCommunityGJRHBatch extends BaseBatch { @Override public void process(Object args) { - def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"]//CcbCommonUtils.citys; + def citys = CcbCommonUtils.citys; def rows = [] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // for (def city : citys) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 596a0c6a6..ec499879a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -16,7 +16,7 @@ import java.sql.ResultSetMetaData @BatchOwner(Watcher.GaoJuan) class ExportDistrictRongHe extends BaseBatch{ - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { runBatch(new ExportDistrictRongHe(), args); @@ -31,20 +31,17 @@ class ExportDistrictRongHe extends BaseBatch{ // exportAllBigdataDealFJW() for(def city : citys) { def cityCode = cityCodeDataMap.get(city) -//// if(!codes.contains(cityCode)){ -//// println city +" "+cityCode -//// } - exportBigdataCommunityGJRH(cityCode) - exportBigdataBuildingFJW(cityCode) - exportBigdataCommfeeGj(cityCode) +// exportBigdataCommunityGJRH(cityCode) +// exportBigdataBuildingFJW(cityCode) +// exportBigdataCommfeeGj(cityCode) // exportBigdataDealFJW(cityCode) - exportBigdataCommbuildstruGJ(cityCode) - exportBigdataCommhousetypeGJ(cityCode) - exportBigdataCommbuildyearGj(cityCode) - exportBigdataCommhouseuseGJ(cityCode) - exportBigdataPositionGJ(cityCode) - exportBigdataDistrictFJW(cityCode) +// exportBigdataCommbuildstruGJ(cityCode) +// exportBigdataCommhousetypeGJ(cityCode) +// exportBigdataCommbuildyearGj(cityCode) +// exportBigdataCommhouseuseGJ(cityCode) +// exportBigdataPositionGJ(cityCode) +// exportBigdataDistrictFJW(cityCode) exportBigdataAliasFJW(cityCode) // // exportBigdataBuildprojFjw(cityCode) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 1549f70c3..ee7114640 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -66,6 +66,8 @@ class CommonDatautils { public static final DBCollection POI_COLL = MongoUtils.getCollectionByName2("fangjia_base", "baidu_place_poi", mongoDbUrl) public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", mongoDbUrl) + public static final DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName2("fangjia", "sqmetre_rent", mongoDbUrl) + public static DSCollection getHouseDataStoreConnection(){ if(house_dataStoreConnection != null) { return house_dataStoreConnection @@ -3097,5 +3099,19 @@ class CommonDatautils { def count = DEAL_PRICE_COLL.count(q); return count; } + + static def findSqmetreRent(def city, def region, def name, def weekend){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + q.put("weekend", weekend) + def dbcur = SQMETRE_RENT_COLL.find(q) + def dbo = null + for(def d : dbcur){ + dbo = d; + } + return dbo; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index e05d00dfa..a1350cc2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,7 +48,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-10-01" + static def statDate = "2021-11-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年11月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2021年12月\\二手房\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 3317161c0..c160a713a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -38,8 +38,8 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", - //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4", - def cityList = ["东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", + //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", + def cityList = [ "青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", From 6d7cdab90fecbd0c19a63fccb3933fc06de0484e Mon Sep 17 00:00:00 2001 From: "2953095102@qq.com" <2953095102@qq.com> Date: Fri, 24 Dec 2021 17:54:24 +0800 Subject: [PATCH 139/212] update --- .../script/ccbevaluation/ExportOracleData.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 4e28e77d6..030a336c2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -29,7 +29,7 @@ class ExportOracleData { // exportCommunityGuidedPrice() // // //exportThirdPartyPrice()//废弃掉,老表 - exportThirdPartyPrice2(202111) +// exportThirdPartyPrice2(202111) // // // @@ -41,7 +41,7 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() - exportBigdataDeal("2021-11-01") +// exportBigdataDeal("2021-11-01") // exportAvmEvaluationGJNB(202111) // exportAVMEvaluationItemGJNB(202111) @@ -52,7 +52,7 @@ class ExportOracleData { // exportCommunityBasePriceGj(202111) -// exportCommunityPdescribeGj(202112) + exportCommunityPdescribeGj(202112) // exportCommCaseDetailGj(202112) // exportLastFile() } @@ -450,7 +450,7 @@ class ExportOracleData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" + fileName = "e:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" // fileName = "D:\\MODEL_000000_000000_BIGDATA_THIRD_PARTY_PRICE_GJ_ADD_${curDate}_0001.dat" } def headers = [] @@ -871,13 +871,13 @@ class ExportOracleData { def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + fileName = "E:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } def headers = [] for(def city : citys) { def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} and cityid='${cityCode}'"; + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where cityid='${cityCode}'";//time=${month} and def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -929,7 +929,7 @@ class ExportOracleData { } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) - println "community_p_describe_gj 上传Ftp完成......" + println "community_p_describe_gj 上传Ftp完成......" } def exportCommCaseDetailGj(month){ From 0fe780b3afdb508b84b20e33fdf2f7e18c035e7e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 18:12:21 +0800 Subject: [PATCH 140/212] update --- .../fangjia/script/ccbevaluation/ExportOracleData.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 4e28e77d6..3c7ee04ac 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -29,7 +29,7 @@ class ExportOracleData { // exportCommunityGuidedPrice() // // //exportThirdPartyPrice()//废弃掉,老表 - exportThirdPartyPrice2(202111) +// exportThirdPartyPrice2(202111) // // // @@ -41,7 +41,7 @@ class ExportOracleData { // exportCommunityBuileType() // exportBigdataDistrictData() // exportBigdataCommunity() - exportBigdataDeal("2021-11-01") +// exportBigdataDeal("2021-11-01") // exportAvmEvaluationGJNB(202111) // exportAVMEvaluationItemGJNB(202111) @@ -53,7 +53,7 @@ class ExportOracleData { // exportCommunityBasePriceGj(202111) // exportCommunityPdescribeGj(202112) -// exportCommCaseDetailGj(202112) + exportCommCaseDetailGj(202112) // exportLastFile() } From c7b88057c8388646d5b9c90d0f4011d440a30c20 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 18:18:07 +0800 Subject: [PATCH 141/212] update --- .../com/fangjia/script/ccbevaluation/ExportOracleData.groovy | 2 ++ 1 file changed, 2 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 3c7ee04ac..d2f8e0d16 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -965,6 +965,8 @@ class ExportOracleData { if(allRows.size()>0){ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); } + + println city } println "comm_case_detail_gj 数据文件输出完成......" From 1fcb15a2f32f9417e21738b3326e50f4e9f6ddff Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 18:20:33 +0800 Subject: [PATCH 142/212] update --- .../com/fangjia/script/ccbevaluation/ExportOracleData.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index d2f8e0d16..b3f63cfec 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -965,7 +965,7 @@ class ExportOracleData { if(allRows.size()>0){ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); } - + println city } From 988ab9834ccc7381dde989beca23ab7ae555018f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 18:50:25 +0800 Subject: [PATCH 143/212] update --- .../script/ccbevaluation/ExportOracleData.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index b3f63cfec..2c6934290 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -866,8 +866,8 @@ class ExportOracleData { def exportCommunityPdescribeGj(month){ - def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -875,9 +875,9 @@ class ExportOracleData { } def headers = [] - for(def city : citys) { - def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month} and cityid='${cityCode}'"; +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -926,7 +926,7 @@ class ExportOracleData { }); println city + " "+allRows.size() FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); - } +// } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" From 8c46e0ecb50455d2b3cad5ebd65cd6e1136b3363 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 18:57:26 +0800 Subject: [PATCH 144/212] update --- .../com/fangjia/script/ccbevaluation/ExportOracleData.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 2c6934290..e525d5d67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -52,8 +52,8 @@ class ExportOracleData { // exportCommunityBasePriceGj(202111) -// exportCommunityPdescribeGj(202112) - exportCommCaseDetailGj(202112) + exportCommunityPdescribeGj(202112) +// exportCommCaseDetailGj(202112) // exportLastFile() } From 74f888f580de9e229deb3719e340685f6ac0a908 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 19:06:15 +0800 Subject: [PATCH 145/212] =?UTF-8?q?=E5=88=86=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ExportOracleData.groovy | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e525d5d67..40e695632 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -866,8 +866,8 @@ class ExportOracleData { def exportCommunityPdescribeGj(month){ -// def citys = CcbCommonUtils.citys; -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -875,9 +875,9 @@ class ExportOracleData { } def headers = [] -// for(def city : citys) { -// def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj"; + for(def city : citys) { + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where cityid='${cityCode}'"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -926,7 +926,7 @@ class ExportOracleData { }); println city + " "+allRows.size() FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); -// } + } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" From f24a782e5bf9c95f76439cc19216d2736e8fa6a8 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 19:08:43 +0800 Subject: [PATCH 146/212] update --- .../script/ccbevaluation/ExportOracleData.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 40e695632..15592c0ef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -866,8 +866,8 @@ class ExportOracleData { def exportCommunityPdescribeGj(month){ - def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -875,9 +875,9 @@ class ExportOracleData { } def headers = [] - for(def city : citys) { - def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where cityid='${cityCode}'"; +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj "; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -924,9 +924,9 @@ class ExportOracleData { data << r.NEIGHBOR_PRICE allRows << data }); - println city + " "+allRows.size() + println allRows.size() FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); - } +// } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" From dee6c01f4181ef4e535207eb8e166c15392b2818 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 24 Dec 2021 19:16:04 +0800 Subject: [PATCH 147/212] =?UTF-8?q?=E6=8C=89=E5=9F=8E=E5=B8=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ExportOracleData.groovy | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 15592c0ef..40e695632 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -866,8 +866,8 @@ class ExportOracleData { def exportCommunityPdescribeGj(month){ -// def citys = CcbCommonUtils.citys; -// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ @@ -875,9 +875,9 @@ class ExportOracleData { } def headers = [] -// for(def city : citys) { -// def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj "; + for(def city : citys) { + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where cityid='${cityCode}'"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -924,9 +924,9 @@ class ExportOracleData { data << r.NEIGHBOR_PRICE allRows << data }); - println allRows.size() + println city + " "+allRows.size() FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); -// } + } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" From 5d796ad2e0c32345d1a4f0df88083156c70a6e3a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Dec 2021 14:54:54 +0800 Subject: [PATCH 148/212] update --- .../ccbevaluation/BigdataBuildingBatch.groovy | 11 ++- .../ccbevaluation/BigdataForsaleBatch.groovy | 13 ++++ .../DealAndThirdPriceDataProcess.groovy | 40 +++++----- .../ccbevaluation/ExportOracleData.groovy | 49 +++++++------ .../fangjia/script/ccbevaluation/Test.groovy | 73 +++++++++++++++---- .../ccbevaluation/data/DistrictMarge.groovy | 4 +- .../ccbevaluation/data/MatchMapData.groovy | 35 +++++++-- .../script/commData/CommonDatautils.groovy | 9 +++ .../gj/district/FangjiaClientUtils.groovy | 2 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 5 +- 10 files changed, 168 insertions(+), 73 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 29e3ac2a4..2fa1ecd76 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.ccb.CcbCodeUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.DateUtils @@ -22,8 +23,7 @@ class BigdataBuildingBatch extends BaseBatch{ @Override public void process(Object args) { - //def citys = CcbCommonUtils.citys; - def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"] + def citys = CcbCommonUtils.citys; def rows = [] def headers = [] for (def city : citys) { @@ -35,9 +35,12 @@ class BigdataBuildingBatch extends BaseBatch{ if(buildingMap){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap) -// rows.addAll(rets) println city + " " + rets.size() - FileUtils.batchCreateFile(rets, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat", headers, "utf-8", true); + def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" + if(CommonDatautils.isWindow()){ + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" + } + FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 3c8299fed..41273e6c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -271,10 +271,23 @@ class BigdataForsaleBatch extends BaseBatch{ if(!seawater.avg_price){ continue; } + if(seawater.name && seawater.name.contains("商住")){ + continue; + } + + if(seawater.estate_type && seawater.estate_type.contains("商住")){ + continue; + } + + if(seawater.purpose && seawater.purpose.contains("商住")){ + continue; + } if(!seawater.height){ seawater.height = null } + + // if(seawater.height && seawater.height<1){ // continue; // } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index 633e69759..e21a8556a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -35,29 +35,29 @@ class DealPriceDataProcess { def thirdMonth = 202111 def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202112_1223 - 副本.xlsx",0,1) for(def d : datas){ -// if(d['G']){ -// def rets = findDealByPe(d['A'], yearMonth, d['G']) -// for(def r : rets){ -// deleteDealPrice(r) -// } -// } - - if(d['H']){ - def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20) - for(def m : rets){ - def precent = d['H']/100.0 - def newPrice = null - if(m.unit_price > m.deal_price){ - newPrice = m.deal_price*precent + m.deal_price - }else if(m.unit_price < m.deal_price){ - newPrice = m.deal_price - (m.deal_price*precent) - } - if(newPrice){ - updateDealPrice(m, newPrice.toInteger()) - } + if(d['G']){ + def rets = findDealByPe(d['A'], yearMonth, d['G']) + for(def r : rets){ + deleteDealPrice(r) } } +// if(d['H']){ +// def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20) +// for(def m : rets){ +// def precent = d['H']/100.0 +// def newPrice = null +// if(m.unit_price > m.deal_price){ +// newPrice = m.deal_price*precent + m.deal_price +// }else if(m.unit_price < m.deal_price){ +// newPrice = m.deal_price - (m.deal_price*precent) +// } +// if(newPrice){ +// updateDealPrice(m, newPrice.toInteger()) +// } +// } +// } + // if(d['M']){ // def rets = findThirdByPe(d['A'], yearMonth, d['M']) // for(def r : rets){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 40e695632..8749dacb7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -24,7 +24,7 @@ class ExportOracleData { } def run() { - +// // exportGuidedPrice() // exportCommunityGuidedPrice() // @@ -43,18 +43,18 @@ class ExportOracleData { // exportBigdataCommunity() // exportBigdataDeal("2021-11-01") -// exportAvmEvaluationGJNB(202111) -// exportAVMEvaluationItemGJNB(202111) -// exportCommunityBasePrice(202111) +// exportAvmEvaluationGJNB(202112) +// exportAVMEvaluationItemGJNB(202112) +// exportCommunityBasePrice(202112) +// +// exportCityPrice(202112) +// exportRegionPrice(202112) -// exportCityPrice(202111) -// exportRegionPrice(202111) +// exportCommunityBasePriceGj(202112) -// exportCommunityBasePriceGj(202111) - - exportCommunityPdescribeGj(202112) +// exportCommunityPdescribeGj(202112) // exportCommCaseDetailGj(202112) -// exportLastFile() + exportLastFile() } @@ -615,7 +615,7 @@ class ExportOracleData { "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; def allRows = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -677,7 +677,7 @@ class ExportOracleData { "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; def allRows = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -725,7 +725,7 @@ class ExportOracleData { def exportCommunityBasePrice(yearMonth){ def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; def allRows = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -768,7 +768,7 @@ class ExportOracleData { def exportCommunityBasePriceGj(yearMonth){ def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; def allRows = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -807,7 +807,7 @@ class ExportOracleData { def exportCityPrice(month){ def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; def allRows = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -836,7 +836,7 @@ class ExportOracleData { def exportRegionPrice(month){ def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; def allRows = [] - sqlInstance2.eachRow(sql,{r-> + sqlInstance.eachRow(sql,{r-> def data = [] data << r.ID data << r.CITYID @@ -866,18 +866,18 @@ class ExportOracleData { def exportCommunityPdescribeGj(month){ - def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" } def headers = [] - - for(def city : citys) { - def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where cityid='${cityCode}'"; +// +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -924,9 +924,9 @@ class ExportOracleData { data << r.NEIGHBOR_PRICE allRows << data }); - println city + " "+allRows.size() + println allRows.size() FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); - } +// } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" @@ -980,6 +980,7 @@ class ExportOracleData { if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 9450808e8..0707c749c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx" -def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01") -def testMap = findCaluateCityPPE(sqlInstance236, 202112) -ExcelUtils.modify(fileName,{d-> - if(d['C']){ - def oldcount = dealMap.get(d['C']) - d['P'] = oldcount - def casecount = testMap.get(d['C']) - d['Q'] = casecount - println d - } -},0) +//def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx" +//def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01") +//def testMap = findCaluateCityPPE(sqlInstance236, 202112) +//ExcelUtils.modify(fileName,{d-> +// if(d['C']){ +// def oldcount = dealMap.get(d['C']) +// d['R'] = oldcount +// def casecount = testMap.get(d['C']) +// d['S'] = casecount +// println d +// } +//},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -271,7 +271,7 @@ ExcelUtils.modify(fileName,{d-> //},0) //def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\第四批需要删除的数据.xlsx",{d-> +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充数据删除3658_1228.xlsx",{d-> // if(d['A']){ // def udata = new BasicDBObject() // udata.put("name_cat", 11) @@ -284,6 +284,53 @@ ExcelUtils.modify(fileName,{d-> // println index // } //},0) +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> + if(d['A']){ + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if(newRegion) { + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if(dist){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", dist._id) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + }else{ + println d + } + } + } +},0) + +//def citys = CcbCommonUtils.citys +//def rets = [] +//for(def city : citys){ +// def count = CommonDatautils.findSeaweeds(city, "district", 1) +// rets << [city, count] +//} +//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\统计.xlsx", rets, "数据") + +//def citys = ["广州","重庆","深圳","上海","北京","南京","泉州"] +//for(def city : citys){ +// def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["name_cat"]) +// def index = 0 +// for(def dist : districts){ +// if(dist.name_cat && dist.name_cat==1){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 0) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dist._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// index++ +// } +// } +// println city +" "+index +//} //ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤的小区.xlsx",{d-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy index 88156feec..d15caa8a7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy @@ -103,7 +103,7 @@ class DistrictMarge { DbUtils.seaweed.update(q, new BasicDBObject('$set', mudata)) updateIndex++ - println "删除:"+updateObj['A'] + println "修改:"+updateObj['A'] + " "+ mudata }else{ println amap_id+" 此组无更新" } @@ -117,7 +117,7 @@ class DistrictMarge { } def getDatas(){ - def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\第四批修正的重复数据_1363.xlsx",0,1) + def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆反查数据_669(审核完毕)(1).xlsx",0,1) def map = [:] for(def d : datas){ if(d['E']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index a1edd0ef0..663546757 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -4,6 +4,7 @@ import com.fangjia.core.RegionData import com.fangjia.core.util.GeoUtils import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils @@ -128,20 +129,23 @@ class MatchMapData { } def run(){ - - def files = ["合肥","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"] + //"合肥", + def files = ["上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"] for(def f : files){ - def city = f + def city = f.replace("-2","") def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365) def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear)) def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend) + def datas = [] + datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型"] ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d-> + def regionCode = null if(d['E']){ def region = d['E'] def newRegion = RegionData.formatRegion(city, region) - if(newRegion){ + regionCode = getRegionCode(newRegion) def dist = CommonDatautils.findDistrict(city, newRegion, d['F'],"district") if(dist){ d['K']="完全一致" @@ -172,6 +176,9 @@ class MatchMapData { if(priceObj){ def markup = (nearDbo.avg_price - priceObj.avg_price)*100/priceObj.avg_price d['M'] = com.fangjia.script.common.DataHandleUtils.getDoubleRound(2, markup) + }else{ +// FangjiaClientUtils.sendDistrictPriceCalculate(dist.city, dist.region, dist.name, 60); +// println dist.city+" "+ dist.region+ " "+ dist.name } } if(nearDbo && sqmetre){ @@ -185,11 +192,25 @@ class MatchMapData { } } + datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O']] + },0) + + ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+city+".xlsx", datas, "数据") } } + def getRegionCode(def region){ + def spell = com.fangjia.core.util.DataUtil.hanyuQuanpin(region, " ") + def arrs = spell.split(" ") + def s = new StringBuffer() + for(def ar : arrs){ + s.append(ar.substring(0,1).toUpperCase()) + } + return s.toString(); + } + def findHouses(def city, def region, def cat, def startDate, def endDate, def name, def alias){ @@ -293,10 +314,10 @@ class MatchMapData { rets << ["建行小区id", "房价网小区id","城市","区域","楼盘名称","单价","月租金","面积","室","厅","卫","出租方式","朝向","楼屋","装修","发布时间"] ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\" + f + ".xlsx", { d -> def flag = d['K'] - if(flag && flag =="完全一致"){ + if(flag && flag =="完全一致" && !d['J']){ def obj = CommonDatautils.findDistrict(city, d['M'], d['N'], "district") if(obj){ - def houses = findHouses3(d['D'], obj._id.toString(), city, d['M'], "rent", 20210801, 20210901, d['N'], null); + def houses = findHouses3(d['D'], obj._id.toString(), city, d['M'], "rent", 20210801, 20211101, d['N'], null); if(houses){ rets.addAll(houses) } @@ -305,7 +326,7 @@ class MatchMapData { } },0) - ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房\\"+city+"_rent.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房2\\"+city+"_rent.xlsx", rets, "数据") } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ee7114640..70c90b65f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1190,6 +1190,15 @@ class CommonDatautils { return obj; } + static def findSeaweeds(city, cat, name_cat){ + DBObject q = new BasicDBObject(); + q.put('city',city) + q.put('cat', cat) + q.put('status',0) + q.put('name_cat',name_cat) + SEAWEED_COLL.count(q) + } + static def findAllSeaweed(def cat , def city){ DBObject q = new BasicDBObject(); q.put('city',city) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index ac634e9e9..e68350d78 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -38,7 +38,7 @@ class FangjiaClientUtils { } public static def sendDistrictPriceCalculate(def city, def region, def name, def count){ - def url = "http://192.168.0.19:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" + def url = "http://192.168.0.100:38080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" // def url = "http://192.168.10.19:8080/FangJiaDataService/districtPriceCalculate?category=sell&city="+city+"&count="+count+"&name="+name+"®ion="+region+"&retainHistory=true" def content = null; try{ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index c160a713a..2bffdeb62 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -38,9 +38,10 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", - //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", + //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158","青岛":"qd|122","南京":"nj|172", + //"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", def cityList = [ - "青岛":"qd|122","南京":"nj|172","嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", + "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", From 7d2c3b480f4a3fd0a1e686940992eb53753aaf48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=B2=88=E5=81=A5?= Date: Tue, 28 Dec 2021 15:15:16 +0800 Subject: [PATCH 149/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E86=E6=97=A5=E5=BF=97?= =?UTF-8?q?=E5=AE=9A=E6=97=B6=E6=B8=85=E7=90=86=E9=9D=9E=E5=BD=93=E6=9C=88?= =?UTF-8?q?=E7=B4=A2=E5=BC=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../batch/RemoveOldIndexOfEs86Batch.groovy | 91 +++++++++++++++++++ .../poi/SeaweedAroundCommittee.groovy | 2 +- .../temp/shenjian/poi/SeaweedAroundPoi.groovy | 2 +- .../poi/SeaweedAroundPoiSingal.groovy | 2 +- .../poi/YinlianLngLatAndAround.groovy | 2 +- 5 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/batch/RemoveOldIndexOfEs86Batch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/batch/RemoveOldIndexOfEs86Batch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/batch/RemoveOldIndexOfEs86Batch.groovy new file mode 100644 index 000000000..076b33977 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/batch/RemoveOldIndexOfEs86Batch.groovy @@ -0,0 +1,91 @@ +package com.fangjia.script.temp.shenjian.batch + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import org.apache.http.client.utils.DateUtils + +import java.util.regex.Matcher +import java.util.regex.Pattern + +/** + * 只保留86机器最近一个月索引 + */ +@BatchOwner(Watcher.ShenJian) +class RemoveOldIndexOfEs86 extends BaseBatch { + + static void main(String[] args) { + runBatch(new RemoveOldIndexOfEs86(), args) + } + + @Override + void process(Object args) { + String url = "http://192.168.0.86:9200/_cat/indices" + + // 必须分开写,不能有空格 + String[] cmds = ["curl", "-XGET", url, "--user", "elastic:goojia123456"] + List> indices = execCurl(cmds) + Pattern yearMonthPattern = Pattern.compile("\\d{6}") + Pattern yearMonthPattern1 = Pattern.compile("\\d{4}\\.\\d{2}") + List indexNeedDel = new ArrayList<>() + for (List index : indices) { + String name = index.get(2) + // 忽略系统索引 + if (name.startsWith(".")) { + continue + } + // 匹配含有年月的索引 + Matcher matcher = yearMonthPattern.matcher(name) + if (matcher.find()) { + String date = matcher.group() + if (dateBeforeCurrMonth(date)) { + indexNeedDel.add(name) + } + continue + } + matcher = yearMonthPattern1.matcher(name) + if (matcher.find()) { + String date = matcher.group() + if (dateBeforeCurrMonth(date)) { + indexNeedDel.add(name) + } + } + } + for (String index : indexNeedDel) { + println "删除索引" + index + String urlPrefix = "http://192.168.0.86:9200/" + cmds = ["curl", "-XDELETE", urlPrefix + index, "--user", "elastic:goojia123456"] + execCurl(cmds) + } + } + + static List> execCurl(String[] cmds) { + ProcessBuilder process = new ProcessBuilder(cmds) + Process p + try { + List result = new ArrayList<>() + p = process.start() + BufferedReader reader = new BufferedReader(new InputStreamReader(p.getInputStream())) + String line = null + while ((line = reader.readLine()) != null) { + result.add(line.split("{1} +")) + } + return result + } catch (IOException e) { + e.printStackTrace() + } + return null + } + + def boolean dateBeforeCurrMonth(String date) { + String dateCurr = DateUtils.formatDate(new Date(), "yyyyMM"); + if (date.contains(dateCurr)) { + return false + } + dateCurr = DateUtils.formatDate(new Date(), "yyyy.MM"); + if (date.contains(dateCurr)) { + return false + } + return true + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy index d8edf839d..fc4fecc64 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundCommittee.groovy @@ -22,7 +22,7 @@ import java.util.concurrent.Executors class SeaweedAroundCommittee { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) - static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy index 7dcd06e51..f3c36ebd8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoi.groovy @@ -28,7 +28,7 @@ import java.util.concurrent.Executors class SeaweedAroundPoi extends BaseBatch { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) - static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static ExecutorService executorService = Executors.newFixedThreadPool(30) static void main(String[] args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy index be70b79e5..1a620af34 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/SeaweedAroundPoiSingal.groovy @@ -28,7 +28,7 @@ import java.util.concurrent.Executors class SeaweedAroundPoiSingal { static DBCollection conn136 = MongoUtils.getCollectionByName("fangjia", "seaweed_poi_sync", "192.168.0.136", 27017) - static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static void main(String[] args) { def types = listAllType() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy index 9cc7ab29d..4764ce68e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/YinlianLngLatAndAround.groovy @@ -14,7 +14,7 @@ import org.apache.commons.collections.CollectionUtils */ class YinlianLngLatAndAround { - static final def URL = "http://192.168.0.19:38080/FangJiaDataService/saveAmapPoi" + static final def URL = "http://192.168.0.100:38080/FangJiaDataService/saveAmapPoi" static void main(String[] args) { this.getAround() From f257b2ee35127f76c483a50dc8a27a14c1f0dba0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Dec 2021 15:47:53 +0800 Subject: [PATCH 150/212] =?UTF-8?q?=E4=B8=8B=E8=BD=BD=E5=90=8E=E6=9C=9F?= =?UTF-8?q?=E6=9C=9B=E4=BB=B7=E4=B8=8D=E5=8F=AF=E5=A1=AB=EF=BC=8C=E6=B7=BB?= =?UTF-8?q?=E5=8A=A0=E4=B8=8B=E8=BD=BD=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/dal/sql/model/FjtEvaluation.java | 12 ++++++++++++ .../weixin/WeiXinEvaluationController.java | 1 + .../src/main/webapp/weiXinEvaluation/bhreport.ftl | 2 +- 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java index c5991df90..8283df958 100644 --- a/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java +++ b/projects/fangjia-dal/src/main/java/com/fangjia/dal/sql/model/FjtEvaluation.java @@ -102,6 +102,8 @@ public class FjtEvaluation implements SqlEntity{ private String finishDateStr;//竣工时间 private String useType;//房屋用途 + + private Date downPdfDate;//下载报告时间 @GeneratedValue(strategy = GenerationType.AUTO) @Id @@ -750,6 +752,16 @@ public class FjtEvaluation implements SqlEntity{ this.useType = useType; } + + @Column(name = "down_pdf_date") + public Date getDownPdfDate() { + return downPdfDate; + } + + public void setDownPdfDate(Date downPdfDate) { + this.downPdfDate = downPdfDate; + } + @Override public String toString() { return "FjtEvaluation{" + diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 64764ab41..3c0a6bc47 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -1025,6 +1025,7 @@ public class WeiXinEvaluationController { String reporNum = "147698"+ String.valueOf(evaluation.getId()); evaluation.setReportNum(reporNum); + evaluation.setDownPdfDate(new Date()); fjtEvaluationService.updateEvaluation(evaluation); try { diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index e70200bda..2c92f982c 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -88,7 +88,7 @@
鏈熸湜浠凤細 -  涓囧厓 + disabled> 涓囧厓 鎻愪氦 鍒锋柊
From d7f3c0401acde805863cabf71a3d1f65fe76ee47 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 28 Dec 2021 17:49:56 +0800 Subject: [PATCH 151/212] update --- .../ccbevaluation/BigdataBuildingBatch.groovy | 4 +- .../ronghe/ExportDistrictRongHe.groovy | 217 ++++++++++-------- .../script/gj/UpdateStatWeekend.groovy | 114 +++++++++ 3 files changed, 232 insertions(+), 103 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/UpdateStatWeekend.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy index 2fa1ecd76..42c580735 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataBuildingBatch.groovy @@ -36,9 +36,9 @@ class BigdataBuildingBatch extends BaseBatch{ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, buildingMap) println city + " " + rets.size() - def fileName = "/home/goojia/ccb/BIGDATA_COMMUNITY_GJ.dat" + def fileName = "/home/goojia/ccb/bigdata_building_fjw.dat" if(CommonDatautils.isWindow()){ - fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\BIGDATA_COMMUNITY_GJ.dat" + fileName = "d:\\数据\\建行\\房产估值\\数据文件\\txt\\bigdata_building_fjw.dat" } FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index ec499879a..c7847b806 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -24,30 +24,29 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { -// def citys = CcbCommonUtils.citys; - def citys = ["苏州","北京","广州","杭州","合肥","深圳","石家庄","武汉","西安","重庆"] + def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() // exportAllBigdataDealFJW() for(def city : citys) { def cityCode = cityCodeDataMap.get(city) -// exportBigdataCommunityGJRH(cityCode) -// exportBigdataBuildingFJW(cityCode) -// exportBigdataCommfeeGj(cityCode) -// - exportBigdataDealFJW(cityCode) -// exportBigdataCommbuildstruGJ(cityCode) -// exportBigdataCommhousetypeGJ(cityCode) -// exportBigdataCommbuildyearGj(cityCode) -// exportBigdataCommhouseuseGJ(cityCode) -// exportBigdataPositionGJ(cityCode) -// exportBigdataDistrictFJW(cityCode) + exportBigdataCommunityGJRH(cityCode) + exportBigdataBuildingFJW(cityCode) + exportBigdataCommfeeGj(cityCode) + + exportBigdataDealFJW(cityCode, '2021-11-01') + exportBigdataCommbuildstruGJ(cityCode) + exportBigdataCommhousetypeGJ(cityCode) + exportBigdataCommbuildyearGj(cityCode) + exportBigdataCommhouseuseGJ(cityCode) + exportBigdataPositionGJ(cityCode) + exportBigdataDistrictFJW(cityCode) exportBigdataAliasFJW(cityCode) -// -// exportBigdataBuildprojFjw(cityCode) -// exportBigdataCommlayoutpicFjw(cityCode) + + exportBigdataBuildprojFjw(cityCode) + exportBigdataCommlayoutpicFjw(cityCode) } -// exportEvaluateRate() + exportEvaluateRate() } @@ -311,94 +310,110 @@ class ExportDistrictRongHe extends BaseBatch{ println "ADDRESS_ALIAS_FJW 上传Ftp完成......" } - def exportBigdataDealFJW(cityCode){ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('2021-06-01','yyyy-MM-dd')"; - def allRows = [] - sqlInstance.eachRow(sql,{r-> - def data = [] - data << r.CITYID - data << r.DEALID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.STREET - data << r.STREETNO - data << r.COMMID - data << r.COMMNAME - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.UNITID - data << r.UNITNO - data << r.UNITNAME - data << r.HOUSEID - data << r.ADDRESS - data << r.BUILDYEAR - data << r.PROPTYPE - data << r.PROPYEARS - data << r.PERIODNUM - data << r.FLOORNUM - data << r.FLOORTYPE - data << r.FLOORCNT - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.BUILDSTRU - data << r.BUILDAREA - data << r.LIVINGAREA - data << r.HOUSESTYLE - data << r.ROOMCNT - data << r.LIVINGROOMCNT - data << r.KITCHENCNT - data << r.BATHROOMCNT - data << r.HOUSEUSE - data << r.HOUSESTRU - data << r.HOUSEASPECT - data << r.FITMENT - data << r.EVALUATENO - data << r.DEALUNITPRICE - data << r.DEALTOTALPRICE - data << r.DECLAREPRICE - data << r.EVALUATEPRICE - data << r.CONTRACTNO - data << r.DEALDATE - data << r.DEALTYPE - data << r.PAYTYPE - data << r.DEALPERIOD - data << r.IFELEVATOR - data << r.COMPAREAGARDEN - data << r.COMPAREATERRACE - data << r.COMPAREABALCONY - data << r.COMPAREABAYWINDOW - data << r.COMPAREAATTIC - data << r.COMPAREAGARAGE - data << r.COMPAREABASEMENT - data << r.COMPAREAOTHER - data << r.DATASOURCE - data << r.AGENCYID - data << r.AGENCYNAME - data << r.REMARK - data << r.CREATETIME1 - data << r.UPDATETIME1 - data << r.SOURCECODE - data << r.WORKDATE - allRows << data - }); + def exportBigdataDealFJW(cityCode, dealdate){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def stop = false def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def index = 0 + while(!stop){ + index++ + def dealStartStr = startDateStr + + def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat" + } + + + def dealEndStr = DateUtils.add(startDateStr, 1) + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${dealStartStr}','yyyy-MM-dd') and dealdate + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + + if(dealStartStr == endDateStr){ + stop = true + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + println "BIGDATA_DEAL_FJW 上传Ftp完成......${dealStartStr}" + startDateStr = dealEndStr } - FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") - println "BIGDATA_DEAL_FJW 上传Ftp完成......" } - - def exportAllBigdataDealFJW(){ def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ"; def allRows = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/UpdateStatWeekend.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/UpdateStatWeekend.groovy new file mode 100644 index 000000000..e231a8d7e --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/UpdateStatWeekend.groovy @@ -0,0 +1,114 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import com.mongodb.DBObject + +import java.text.ParseException +import java.text.SimpleDateFormat +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + +class UpdateStatWeekend { + + public static final def mongoDbUrl = "mongodb://developer:G00IDCDEV12385@mongo.db.fangjia.com:28017"; + public static final DBCollection DIST_STAT_COLL = MongoUtils.getCollectionByName2("fangjia_stat", "district_stat", mongoDbUrl) + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(80) + + static main(args) { + def task = new UpdateStatWeekend() + task.run() + } + + def run(){ + //"合肥","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆", + def citys = ["苏州"] + for(def city : citys){ + def rets = findAddData(city) + process(city, rets) + } + } + + def process(def city, def rets){ + + def index = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(rets.size()) + + for(def i=0;i Date: Fri, 31 Dec 2021 13:53:45 +0800 Subject: [PATCH 152/212] =?UTF-8?q?=E5=8E=BB=E7=A9=BA=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/fangjia/open/util/BHBankUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java index 1e18831b4..2c115f3b4 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/util/BHBankUtils.java @@ -441,7 +441,7 @@ public class BHBankUtils { Map params = new HashMap(); params.put("queryId",queryId); if(StringUtils.isNotBlank(reportMsg)){ - params.put("reportMsg",reportMsg); + params.put("reportMsg",reportMsg.replace(" ","")); } From c036d5d677844bb548379b09aadf81890b9f7b8e Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 31 Dec 2021 14:15:10 +0800 Subject: [PATCH 153/212] =?UTF-8?q?=E6=8E=A5=E5=8F=A3=E5=8E=BB=E7=A9=BA?= =?UTF-8?q?=E6=A0=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/open/controllers/PropertyController.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java index 1ba157ee5..f6e9855f0 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/PropertyController.java @@ -2363,7 +2363,7 @@ public class PropertyController { mav.setModel(BasicModel.buildBadRequestModel("size(面积)不能为空")); return mav; } - District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName(), + District district = districtService.findByNameAndAlias(model.getCity(), model.getDistrict(), model.getName().replace(" ",""), "district"); if (null == district) { mav.setModel(BasicModel.buildBadRequestModel("该小区不存在")); From be88ca7c64e82e678df953d22d8a470cfc9baf50 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 5 Jan 2022 11:34:01 +0800 Subject: [PATCH 154/212] =?UTF-8?q?=E5=AF=BC=E5=87=BA=E5=B0=8F=E5=8C=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 4 +- .../ccbevaluation/data/MatchMapData.groovy | 127 +++++++++++++++++- .../com/fangjia/script/gj/test/Test.groovy | 41 +++--- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 12 +- .../TestCrawlTotalCount.groovy | 4 +- 5 files changed, 158 insertions(+), 30 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 8749dacb7..e86a065cc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -40,7 +40,7 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() -// exportBigdataCommunity() + exportBigdataCommunity() // exportBigdataDeal("2021-11-01") // exportAvmEvaluationGJNB(202112) @@ -54,7 +54,7 @@ class ExportOracleData { // exportCommunityPdescribeGj(202112) // exportCommCaseDetailGj(202112) - exportLastFile() +// exportLastFile() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index 663546757..51ea1deba 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -10,7 +10,9 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.poi.ss.util.RegionUtil import org.elasticsearch.index.query.BoolQueryBuilder @@ -21,13 +23,17 @@ class MatchMapData { def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol(); + public static final def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; + public static final DBCollection ZS_LEVEL_CONN = MongoUtils.getCollectionByName2("fangjia", "district_analy", ZS_MONGODB_URL) + static main(args) { def task = new MatchMapData() - task.run() +// task.run() // task.run1() // task.run2() // task.run3() // task.exportHouse() + task.addDistrictLevel() } @@ -306,7 +312,8 @@ class MatchMapData { } def exportHouse(){ - def files = ["合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆","苏州"] + //"合肥","石家庄","北京","广州","杭州","上海","深圳","天津","武汉","西安","重庆", + def files = ["苏州","合肥"] for(def f : files) { def city = f @@ -326,7 +333,7 @@ class MatchMapData { } },0) - ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房2\\"+city+"_rent.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\融合\\租房3\\"+city+"_rent.xlsx", rets, "数据") } } @@ -338,7 +345,8 @@ class MatchMapData { QueryBuilder area_range = QueryBuilders.rangeQuery("area").from(20).to(500).includeLower(true).includeUpper(false); QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("city", city)).must(QueryBuilders.termQuery("region", region)).must(QueryBuilders.termQuery("cat", cat)) - .must(QueryBuilders.termQuery("root", "lianjia.com")).must(date_range).must(area_range) + .must(date_range).must(area_range) + //.must(QueryBuilders.termQuery("root", "lianjia.com")) if(!alias){ alias = [] @@ -376,5 +384,116 @@ class MatchMapData { return rets; } + def addDistrictLevel(){ + def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"] + + def dataMap = null + + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + + for(def fn : files){ + def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend) + def datas = addPriceData(dists, priceMap); + def levelMap = calcuteLevel(datas); + + dataMap = getDataMap(fn); + + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d-> + + def region = dataMap.get(d['C']) + if(region){ + def newRegion = RegionData.formatRegion(fn, region) + if(newRegion){ + def level = levelMap.get( fn+"##"+newRegion+"##"+d['D']) + if(level){ + d['J'] = level + } + } + } + + },0) + } + } + + def addPriceData(dists, priceMap){ + def newLists = [] + for(def d : dists){ + def price = priceMap.get(d.city+"##"+d.region+"##"+d.name); + if(price){ + d.avg_price = price; + newLists << d; + } + } + if(newLists){ + newLists.sort {a,b-> + def v1 = a.avg_price + def v2 = b.avg_price + return v1.compareTo(v2) + } + } + return newLists; + } + + def calcuteLevel(def datas){ + def total = datas.size() + def v = (int)(total/10) + def start = 0; + def end = 0; + def map = [:] + for(int i=1;i<=10;i++){ + end = end + v; + if(i == 10){ + map.put(i, [start, total]) + }else{ + map.put(i, [start, end]) + } + start = end; + } + for(def d=0;dm && d<=n){ + dist.level = 10 - o.key + 1 + break + } + } + } + } + def dataMap = [:] + for(def d : datas){ + dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level) + } + return dataMap; + } + + def getDataMap(city){ + def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+city+".xlsx",0,1) + def map = [:] + for(def d : datas){ + map.put(d['D'], d['E']) + } + return map + + } + def getDistrictLevel(city, region, name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", name) + return ZS_LEVEL_CONN.findOne(q) + } + } + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy index 91498750a..728e6510c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/Test.groovy @@ -2,6 +2,7 @@ package com.fangjia.script.gj.test import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject @@ -29,25 +30,33 @@ import java.text.SimpleDateFormat // } //}) -def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); +//def onLineSqlInstance = Sql.newInstance("jdbc:mysql://mysql.db.fangjia.com:4306/fangjia_youfang?&characterEncoding=utf8", "sa", "G00jia", "com.mysql.jdbc.Driver"); +// +//def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; +//def countMap = [:] +//onLineSqlInstance.eachRow(sql,{r-> +// def addTime = r.visit_time +// def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") +// def ips = countMap.get(month) +// if(!ips){ +// ips = new HashSet(); +// countMap.put(month, ips) +// } +// ips << r.ip +//}); +// +//countMap.each {k,v-> +// println k+" "+v.size() +//} -def sql = "select visit_time,ip from visit_log where visit_time>='2020-09-18'"; -def countMap = [:] -onLineSqlInstance.eachRow(sql,{r-> - def addTime = r.visit_time - def month = DateUtils.formatDateByPattern(addTime, "yyyy-MM") - def ips = countMap.get(month) - if(!ips){ - ips = new HashSet(); - countMap.put(month, ips) - } - ips << r.ip -}); -countMap.each {k,v-> - println k+" "+v.size() +def dists = CommonDatautils.findAllDistrictByCity("上海", "district", ["city","region","name","finish_building_date"]) +def rets = [] +for(def d : dists){ + def year = CommonUtils.getDistrictAge(d) + rets << [d.city,d.region, d.name, year] } - +ExcelUtils.write("D:\\数据\\统计房龄.xlsx", rets, "数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 2bffdeb62..d369ef53a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -37,12 +37,12 @@ class Esf58Crawl extends BaseBatch{ def static final distMap = [:] def static useDistKeys = [] static def totalCount = new AtomicLong() - //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102","上海":"sh|2", "北京":"bj|1","天津":"tj|18", - //"重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158","青岛":"qd|122","南京":"nj|172", - //"嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669", - def cityList = [ - - "沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", + //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", +// "上海":"sh|2", "北京":"bj|1","天津":"tj|18", +// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", + def cityList = ["青岛":"qd|122","南京":"nj|172", + "嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", + "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index d28d0da9d..a03f7ea2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202110); - def count2 = findTotalCount(source, 202111); + def count1 = findTotalCount(source, 202111); + def count2 = findTotalCount(source, 202112); println source +" "+count1 +" "+count2 } } From 7bddc42298adb6a84e6311e3f089f91bd2d81460 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Jan 2022 12:20:33 +0800 Subject: [PATCH 155/212] update --- .../ccbevaluation/ExportOracleData.groovy | 6 +- .../ccbevaluation/data/ExcelDataTest.groovy | 127 ++++++ .../ccbevaluation/data/MatchMapData.groovy | 60 ++- .../ccbevaluation/ronghe/SSHHelper.groovy | 2 +- .../script/commData/CommonDatautils.groovy | 4 +- .../script/gj/level/DistrictIndexBatch.groovy | 420 ++++++++++++++++++ .../script/thirdPriceCrawl/Esf58Crawl.groovy | 65 ++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 3 +- 8 files changed, 657 insertions(+), 30 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e86a065cc..24dbf7876 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -76,7 +76,7 @@ class ExportOracleData { }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; - def fileName = "/output/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_EVALUATE_RATE_GJ_ALL_${curDate}_0001.dat" } @@ -503,7 +503,7 @@ class ExportOracleData { allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ.dat" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_COMMUNITY_GJ_ALL_${curDate}_0001.dat" } @@ -522,7 +522,7 @@ class ExportOracleData { def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; - def fileName = "/home/goojia/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" + def fileName = "/ccb/MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_BIGDATA_FORSALE_GJ_ADD_${curDate}_0001.dat" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy new file mode 100644 index 000000000..8373b928c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -0,0 +1,127 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils + +//def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1) +//def cityDataMap = [:] +//for(def row : cityDatas){ +// if(row['D']){ +// cityDataMap.put(row['A']+"##"+row['B']+"##"+row['C'], row) +// } +//} +// +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据_1529.xlsx",{d-> +// +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// def key = d['A']+"##"+newRegion+"##"+d['C'] +// println key +// def datas = cityDataMap.get(key) +// if(datas){ +// d['F'] = datas['D'] +// d['G'] = datas['E'] +// d['H'] = datas['F'] +// d['I'] = datas['G'] +// d['J'] = datas['H'] +// +// } +//},0) + +def city = "广州" +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// def newRegion = RegionData.formatRegion(city, d['A']) +// if(newRegion){ +// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null); +// if(rMap){ +// d['C'] = rMap.region +// d['D'] = rMap.name +// } +// } +// +//},0) + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",{d-> +// def newRegion = RegionData.formatRegion(city, d['E']) +// if(newRegion){ +// def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['F'], null); +// if(rMap){ +// d['L'] = rMap.region +// d['M'] = rMap.name +// } +// } +//},0) + +//def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\融合\\广州 - 副本.xlsx",0, 1) +//def idMaps = [:] +//for(def d : datas){ +// if(d['L']){ +// idMaps.put(d['L']+"##"+d['M'], d) +// } +//} +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// if(d['C']){ +// def key = d['C']+"##"+d['D'] +// def rows = idMaps.get(key) +// if(rows){ +// d['E'] = rows['D'] +// d['F'] = rows['E'] +// d['G'] = rows['F'] +// } +// } +//},0) + + +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district") +// if(obj){ +// def year = CommonUtils.getDistrictAge(obj) +// if(year){ +// d['H'] = year +// } +// if(obj.alias){ +// d['K'] = obj.alias.join(" ") +// } +// +// } +//},0) + +def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1) +def scoreMap = [:] +for(def d : data1s){ + scoreMap.put(d['B'], d) +} + +def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1) +def levelMap = [:] +for(def d : data2s){ + levelMap.put(d['E'], d) +} + +ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> + if(d['E']){ + def row = scoreMap.get(d['E']) + if(row){ + d['I'] = row['G'] + d['J'] = row['H'] + d['K'] = row['I'] + d['L'] = row['J'] + d['M'] = row['K'] + d['N'] = row['L'] + d['O'] = row['M'] + } + + def data = levelMap.get(d['E']) + if(data){ + d['P'] = data['G'] + d['Q'] = data['H'] + d['R'] = data['I'] + d['S'] = data['J'] + d['T'] = data['K'] + d['U'] = data['L'] + d['V'] = data['M'] + } + } +},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index 51ea1deba..5e58e7b38 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -19,6 +19,8 @@ import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders +import java.util.concurrent.CopyOnWriteArrayList + class MatchMapData { def thirdPartyColl = com.fangjia.script.thirdPriceCrawl.CrawalDBUtil.getThirdPartyCol(); @@ -28,12 +30,12 @@ class MatchMapData { static main(args) { def task = new MatchMapData() -// task.run() + task.run() // task.run1() // task.run2() // task.run3() // task.exportHouse() - task.addDistrictLevel() +// task.addDistrictLevel() } @@ -135,16 +137,22 @@ class MatchMapData { } def run(){ - //"合肥", - def files = ["上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"] + //"广州", + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def files = ["合肥","广州","上海","北京","杭州","深圳","石家庄","天津","武汉","西安","重庆","苏州"] for(def f : files){ def city = f.replace("-2","") def lastYear = DateUtils.add(DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd"), -365) def lastWeekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(lastYear)) def districtPriceMap = CommonDatautils.queryAllDistrictPriceByWeekend(city, lastWeekend) - def datas = [] - datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型"] + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city", "region","name"]) + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def priceDatas = addPriceData(dists, priceMap); + def levelMap = calcuteLevel(priceDatas); + + def datas = new CopyOnWriteArrayList() + datas << ["城市编码","区域编码","ID","小区名","Wgs85经度","Wgs84纬度","租售比","小区涨跌幅","物业类型","评级","均价"] ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\"+f+".xlsx",{d-> def regionCode = null if(d['E']){ @@ -192,13 +200,23 @@ class MatchMapData { d['N'] = prc.toInteger() } d['O'] = dist.estate_type2 + + + def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name) + if(dist.name == "绿苑公寓"){ + println dbo + } + if(dbo){ + d['P'] = dbo.level + d['Q'] = dbo.avg_price + } } }else{ d['M'] = "区域无法标准化" } } - datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O']] + datas << [d['B'], regionCode, d['D'], d['F'], d['I'], d['H'], d['N'], d['M'], d['O'], d['P'], d['Q']] },0) @@ -385,12 +403,12 @@ class MatchMapData { } def addDistrictLevel(){ +// def files = ["合肥","苏州","上海","北京","广州","杭州","深圳","石家庄","天津","武汉","西安","重庆"] def dataMap = null def weekend = DateUtils.getFirstDayOfWeek(new Date()); - for(def fn : files){ def dists = CommonDatautils.findAllDistrictByCity(fn, "district", ["city", "region","name"]) def priceMap = CommonDatautils.queryDistrictPrice(fn, weekend) @@ -399,15 +417,28 @@ class MatchMapData { dataMap = getDataMap(fn); - ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\详情2\\"+fn+".xlsx",{d-> + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\融合\\租售比\\评级10级\\"+fn+".xlsx",{d-> def region = dataMap.get(d['C']) if(region){ def newRegion = RegionData.formatRegion(fn, region) if(newRegion){ - def level = levelMap.get( fn+"##"+newRegion+"##"+d['D']) - if(level){ - d['J'] = level + def dist = CommonDatautils.findDistrict(fn, newRegion, d['D'],"district") + if(!dist){ + dist = CommonDatautils.findSeaweedByMappingAlias(fn, newRegion, d['D'],"district") + if(!dist){ + def rets = CommonDatautils.findDistrictByAlias(fn, newRegion, d['D'],"district") + if(rets){ + dist = rets[0] + } + } + } + if(dist){ + def dbo = levelMap.get(dist.city+"##"+dist.region+"##"+dist.name) + if(dbo){ + d['J'] = dbo.level + d['K'] = dbo.avg_price + } } } } @@ -472,7 +503,10 @@ class MatchMapData { } def dataMap = [:] for(def d : datas){ - dataMap.put(d.city+"##"+d.region+"##"+d.name, d.level) + if(d.name=="绿苑公寓"){ + println d + } + dataMap.put(d.city+"##"+d.region+"##"+d.name, d) } return dataMap; } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy index af4571228..705a5c1f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/SSHHelper.groovy @@ -105,7 +105,7 @@ class SSHHelper { public static void main(String[] args) { SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20211210")); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh 20220110")); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 70c90b65f..23945eac2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -705,7 +705,7 @@ class CommonDatautils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","长隆小区") +// q.put("name","吴中路9弄小区") DBObject keys = new BasicDBObject(); if(fields){ @@ -829,6 +829,8 @@ class CommonDatautils { keys.put("lat", 1); keys.put("lng", 1); keys.put("name_cat", 1); + keys.put("name_cat", 1); + keys.put("name_cat", 1); DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy new file mode 100644 index 000000000..e901f862b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -0,0 +1,420 @@ +package com.fangjia.script.gj.level + +import com.fangjia.core.util.HttpRequest +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.BaseBatch +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.JacksonUtil +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.apache.commons.httpclient.util.DateUtil +import org.apache.commons.lang.StringUtils +import org.bson.types.ObjectId +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.ResultSet +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong +import java.util.zip.GZIPInputStream + +class DistrictIndexBatch extends BaseBatch{ + + public static EsDataSource dataSource = new EsDataSource() + private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc" + + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + + static main(args) { + runBatch(new DistrictIndexBatch(), args); + } + + @Override + public void process(Object args) { + run(); + } + + def run(){ + def citys = ["上海"]//CommonDatautils.findAllCity(); + for(def city : citys){ + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) + calcute(city, dists) + } + } + + def calcute(city, dists){ + def weekend = DateUtils.getFirstDayOfWeek(new Date()); + def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) + def costDatas = getCostByCity(city) + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def datas = new CopyOnWriteArrayList() + datas << ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] + for(def i=0;i list = DbUtils.findAroundDistrict(d) + def avg_charge = null + def rank = null + if (list) { + list = sortByDistance(d, list); + avg_charge = gtAvgEstateCharge(list) + rank = getPriceInRegionRank(d, list, priceMap) + } + def total_price = getHousePrice(d, dists, priceMap); + def avg_cost = getAvgCost(d, costDatas) + + def total_price_index = 0 + if (total_price) { + total_price_index = 286.0134196 * (total_price - 531.2615913) / 313.3693508 + } + def avg_charge_index = 0 + if (avg_charge) { + avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241 + } + + def rank_index = 0 + if (rank) { + rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636 + } + + def avg_cost_index = 0 + if (avg_cost) { + avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959 + } + def potentiality = avg_cost_index + 4099.978248 + potentiality = DataHandleUtils.getDoubleRound(2, potentiality) + def map = [:] + map.id = new ObjectId().toString() + map.city = d.city + map.region = d.region + map.name = d.name + map.avg_total_price = total_price + map.avg_estate_charge = avg_charge + map.price_rank = rank + map.avg_cost = avg_cost + map.comp_value = potentiality + map.month = "2022-01" + map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") + + datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] + + def newUrl = targetUrl +"/"+map.id + httpSaveData(newUrl, map) + println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown(); + } + } + }) + } + latch.await(); + // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") + } + + def getHousePrice(d, dists, priceMap){ + def areas = findHouse(d, 20200101, 20220101) + def avg_area = null + if(areas){ + avg_area = areas.sum()/areas.size(); + }else{ + if(d.finish_building_date){ + def year = CommonUtils.getBuildYear(d.finish_building_date) + if(year){ + def startYear = (year.toString().substring(0,3)+"0").toInteger() + def endYear = startYear + 10 + + def selectDists = [] + for(def ad : dists){ + if(d.region == ad.region){ + def adYear = CommonUtils.getBuildYear(ad.finish_building_date) + if(adYear){ + adYear = adYear.toInteger() + if(adYear>=startYear && adYear<=endYear){ + selectDists << ad + } + } + } + } + + if(selectDists){ + def allAreas = [] + for(def sd : selectDists){ + def areaDatas = findHouse(sd, 20190101, 20220101) + allAreas.addAll(areaDatas) + } + avg_area = allAreas.sum()/allAreas.size(); + } + } + } + } + + if(avg_area){ + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) + def total_price = (avg_price * avg_area)/10000.0 + def total = DataHandleUtils.getDoubleRound(2, total_price) + return total + } + return null + } + + + def gtAvgEstateCharge(list){ + def rets = [] + for(def i=0;i=3){ + break; + } + } + } + def avg_charge = rets.sum()/rets.size(); + avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge) + return avg_charge; + } + + def getPriceInRegionRank(d, nbLists, priceMap){ + if(nbLists){ + if(nbLists.size()==1){ + return 0.5; + }else{ + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) + def count = 0 + if(avg_price){ + for(def nb : nbLists){ + def price = priceMap.get(nb.city+"##"+nb.region+"##"+nb.name) + if(price > avg_price){ + count ++ + } + } + } + def pre = (nbLists.size() - count)/nbLists.size() + pre = DataHandleUtils.getDoubleRound(2, pre) + return pre; + } + } + return null; + } + + def getCostByCity(city){ + def datas = queryPoi(city) + return datas + } + + def sortByDistance(d, list){ + for(def nb : list){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, nb.lat/10000000000.0, nb.lng/10000000000.0) + nb.distance = distance + } + //排序 + Collections.sort(list, new Comparator() { + @Override + int compare(Object o1, Object o2) {return o1.distance-o2.distance;} + }) + return list; + } + + def getAvgCost(d, costDatas){ + if(costDatas){ + def costs = [] + for(def c : costDatas){ + def distance = GeoHelper.GetDistance(d.lat/10000000000.0, d.lng/10000000000.0, c.lat, c.lng) + if(distance <= 1000){ + costs << c.cost + } + } + if(costs){ + def avg_cost = costs.sum()/costs.size() + avg_cost = DataHandleUtils.getDoubleRound(2, avg_cost) + return avg_cost + } + + } + return null + } + + + def queryPoi(def city){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode,business.keytag as keytag,business.cost as cost FROM amap_poi_detail WHERE 1=1") + sqlBuffer.append(" and cityname = '" + city + "市'") + sqlBuffer.append(" and business.cost>20 and business.cost<500") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def datas = [] + def tags = ["法国菜","东南亚菜","日本料理","火锅","海鲜","自助餐","西餐","浙菜","茶餐厅","酒吧","烤肉","潮州菜","云贵菜","韩国料理","中餐","特色火锅","西北菜","串串香","农家乐","烧烤","茶座","上海菜","意大利菜","素菜馆","蛋糕店","东北菜","湘菜","粤菜","咖啡厅","轰趴馆","清真菜","北京菜","川菜","餐饮服务,餐饮相关场所,餐饮相关"] + def index = 0 + while (rs.next()) { + index++ + def tag = rs.getString("keytag") + if(tags.contains(tag)){ + def dMap = [:] + dMap.cost = rs.getInt("cost") + def location = rs.getObject("location") + if(location){ + dMap.lng = location.x + dMap.lat = location.y + } + datas << dMap + }else{ + println tag + } + } + println index + return datas; + } + + + def findHouse(def dist, def startDate, def endDate){ + def nameQuery = getAllName(dist); + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT area FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+dist.city+"'") + sqlBuffer.append(" and region = '"+dist.region+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + sqlBuffer.append(" and (" + nameQuery + ")"); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def areas = [] + while (rs.next()){ + areas << rs.getDouble("area") + } + return areas; + } + + def getAllName(dist){ + Set aliasNames = new LinkedHashSet(); + aliasNames.add(dist.name); + if (dist != null && dist.alias != null) { + aliasNames.addAll(dist.alias); + } + + List namePars = new ArrayList(); + if(aliasNames!=null && aliasNames.size()>0) { + for(String name : aliasNames){ + if(name && name.trim().length() > 1){ + namePars.add("district_name ='" + name + "'"); + } + } + } + + if(namePars.size() > 0){ + return namePars.join(" OR ") + } + return null; + } + + + +// def updateHouseMatchDistrictId(def id, def match_id){ +// def header = [:] +// header["Authorization"] = "Basic ${Base64.getEncoder().encodeToString("developer:5e8edd851d2fdfbd7415232c67367cc3".getBytes("UTF-8"))}".toString() +// String seawaterUrl = "http://192.168.0.206:9200/seawater-2021/_update_by_query?conflicts=proceed&scroll_size=1000&wait_for_completion=false" +// def body = """ +// { +// "script": { +// "source": "ctx._source.match_seaweed_id='${match_id}'", +// "lang": "painless" +// }, +// "query": { +// "term": { +// "_id": "${id}" +// } +// } +// } +// """ +// HttpRequest request = new HttpRequest() +// String s = request.postWithBody(seawaterUrl, JSONObject.fromObject(body.toString()), header, "UTF-8") +// println s +// } + + + + def httpSaveData(def newUrl,def params){ + def header = [:] + header.put("Authorization","Basic ZWxhc3RpYzpnb29qaWExMjM0NTY=") + doPost(newUrl,header, params, true, "utf-8") + } + + + def doPost(def targetUrl, def headers, def params, def isJson, def charset){ + try{ + URL url = new URL(targetUrl); + HttpURLConnection conn = (HttpURLConnection) url.openConnection(); + conn.setRequestMethod("POST"); + conn.setDoOutput(true); + conn.setUseCaches(false); + conn.setConnectTimeout(5000); + conn.setReadTimeout(5000); + conn.setRequestProperty("User-Agent", UserAgents.get()); + String param = null; + BufferedWriter mWriter = null; + InputStreamReader inStream = null; + StringBuilder buffer = new StringBuilder(); + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + conn.setRequestProperty(head, headers.get(head)); + } + } else { + conn.setRequestProperty("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8" ); + } + + if (isJson) { + conn.setRequestProperty("Content-Type", "application/json; charset=" + charset); + param = JacksonUtil.beanToJson(params) + } + + conn.connect(); + + //println param.toString() + //输出流 + if (StringUtils.isNotEmpty(param)) { + mWriter = new BufferedWriter(new OutputStreamWriter(conn.getOutputStream(), charset)); + mWriter.write(param); + mWriter.flush(); + mWriter.close(); + } + if (conn.getHeaderField("Content-Encoding") != null + && conn.getHeaderField("Content-Encoding").equals("gzip")) { + inStream = new InputStreamReader(new GZIPInputStream(conn.getInputStream()), charset); + } else { + inStream = new InputStreamReader(conn.getInputStream(), charset); + } + char[] bf = new char[1024]; + int len; + while ((len = inStream.read(bf)) != -1) { + buffer.append(bf, 0, len); + } + } catch (Exception e) { + e.printStackTrace() + } + return null; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index d369ef53a..ae2639418 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -38,9 +38,10 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = [] static def totalCount = new AtomicLong() //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", -// "上海":"sh|2", "北京":"bj|1","天津":"tj|18", +// "青岛":"qd|122", // "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", - def cityList = ["青岛":"qd|122","南京":"nj|172", +// "上海":"sh|2", + def cityList = [ "北京":"bj|1","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", @@ -261,7 +262,7 @@ class Esf58Crawl extends BaseBatch{ def newUrl = url+"?sort=param62_asc&key="+URLEncoder.encode(keyword, "utf-8") def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET); if(document){ - Elements elements = JQueryUtils.select([".list-cell a"], document) + Elements elements = JQueryUtils.select([".list-cell .li-row"], document) if(elements){ addSearchContent(city, url, keyword); }else{ @@ -279,9 +280,11 @@ class Esf58Crawl extends BaseBatch{ def document = CrawalUtil.getDocumentByABuYun(newUrl, ERROR_MSG, CHARSET); println "[${totalCount.get()}]关键字【${keyword}】 页:"+ newUrl if(document){ - Elements elements = JQueryUtils.select([".list-cell a"], document) - if(elements){ - saveDetailContent(city, document); + Elements infoElements = JQueryUtils.select([".list-cell .li-info"], document) + Elements priceElements = JQueryUtils.select([".list-cell .li-side"], document) + if(infoElements){ +// saveDetailContent(city, document); + saveLists(city, infoElements, priceElements) }else{ flag = false; } @@ -289,6 +292,46 @@ class Esf58Crawl extends BaseBatch{ flag = false; } page++; + if(page>50){ + flag = false; + } + } + } + + + def saveLists(city, infoElements, priceElements){ + for(def i=0;i") } - def avg_price = JQueryUtils.selectText(".price", detailDocument); + def avg_price = JQueryUtils.selectText(".average", detailDocument); def region; - def addr = JQueryUtils.selectText("td:contains(详细地址)+td", detailDocument); + def addr = JQueryUtils.selectText(".column-1:contains(小区地址) .value", detailDocument); if(info){ if(info.size()>=4){ - city = info[2].replace("小区","").trim() - region = info[3].replace("小区","").trim() + city = info[1].replace("小区","").trim() + region = info[2].replace("小区","").trim() } } if(!addr){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index c106ca013..3993f6663 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,7 +17,8 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["fang.com":"搜房","cityhouse.cn":"城市房产","5i5j.com":"我爱我家"] + def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多", + "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] static main(args) { def task = new SynchDistrictPrice() task.run() From 1be866e02f3764811bd0cbf17105509cb9e3a010 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Jan 2022 12:54:04 +0800 Subject: [PATCH 156/212] update --- .../com/fangjia/script/gj/level/DistrictIndexBatch.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index e901f862b..66bc98f49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -26,6 +26,9 @@ import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong import java.util.zip.GZIPInputStream +/** + * 小区潜力指数 + */ class DistrictIndexBatch extends BaseBatch{ public static EsDataSource dataSource = new EsDataSource() From d0d70e6681499696e89119d3e03fd2de165b11dd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Jan 2022 13:30:37 +0800 Subject: [PATCH 157/212] update --- .../script/gj/level/DistrictIndexBatch.groovy | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 66bc98f49..f1d9f31c7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -5,6 +5,7 @@ import com.fangjia.httpclient.UserAgents import com.fangjia.script.BaseBatch import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils @@ -43,7 +44,7 @@ class DistrictIndexBatch extends BaseBatch{ static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) static main(args) { runBatch(new DistrictIndexBatch(), args); @@ -58,6 +59,8 @@ class DistrictIndexBatch extends BaseBatch{ def citys = ["上海"]//CommonDatautils.findAllCity(); for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) + + dists = dists[0..20] calcute(city, dists) } } @@ -91,23 +94,27 @@ class DistrictIndexBatch extends BaseBatch{ def total_price_index = 0 if (total_price) { total_price_index = 286.0134196 * (total_price - 531.2615913) / 313.3693508 + total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index) } def avg_charge_index = 0 if (avg_charge) { avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241 + avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index) } def rank_index = 0 if (rank) { rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636 + rank_index = DataHandleUtils.getDoubleRound(6, rank_index) } def avg_cost_index = 0 if (avg_cost) { avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959 + avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index) } def potentiality = avg_cost_index + 4099.978248 - potentiality = DataHandleUtils.getDoubleRound(2, potentiality) + potentiality = DataHandleUtils.getDoubleRound(6, potentiality) def map = [:] map.id = new ObjectId().toString() map.city = d.city @@ -124,7 +131,7 @@ class DistrictIndexBatch extends BaseBatch{ datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] def newUrl = targetUrl +"/"+map.id - httpSaveData(newUrl, map) + // httpSaveData(newUrl, map) println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price } catch (Exception e) { e.printStackTrace() @@ -135,6 +142,15 @@ class DistrictIndexBatch extends BaseBatch{ }) } latch.await(); + + + def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] + def fileName = "/home/gj/potentiality.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\评级\\上海综合潜力.dat" + } + + FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") } @@ -169,7 +185,10 @@ class DistrictIndexBatch extends BaseBatch{ def areaDatas = findHouse(sd, 20190101, 20220101) allAreas.addAll(areaDatas) } - avg_area = allAreas.sum()/allAreas.size(); + if(allAreas){ + avg_area = allAreas.sum()/allAreas.size(); + } + } } } From 2c2dc926fe47332bc6addf6e7a23b05f24945888 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Jan 2022 13:34:15 +0800 Subject: [PATCH 158/212] update --- .../com/fangjia/script/gj/level/DistrictIndexBatch.groovy | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index f1d9f31c7..aa68defee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -44,7 +44,7 @@ class DistrictIndexBatch extends BaseBatch{ static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(1) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) static main(args) { runBatch(new DistrictIndexBatch(), args); @@ -59,8 +59,7 @@ class DistrictIndexBatch extends BaseBatch{ def citys = ["上海"]//CommonDatautils.findAllCity(); for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) - - dists = dists[0..20] +// dists = dists[0..20] calcute(city, dists) } } From f2cbbbece93d9f078f71e87981e42c14cb1bd04f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Jan 2022 15:33:37 +0800 Subject: [PATCH 159/212] =?UTF-8?q?=E9=80=9F=E5=BA=A6=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/level/DistrictIndexBatch.groovy | 47 ++++++++++++++++--- 1 file changed, 40 insertions(+), 7 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index aa68defee..44fe7b006 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -13,6 +13,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.JacksonUtil import com.mongodb.DBObject +import groovy.sql.Sql import net.sf.json.JSONObject import org.apache.commons.httpclient.util.DateUtil import org.apache.commons.lang.StringUtils @@ -46,6 +47,9 @@ class DistrictIndexBatch extends BaseBatch{ public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { runBatch(new DistrictIndexBatch(), args); } @@ -112,10 +116,10 @@ class DistrictIndexBatch extends BaseBatch{ avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959 avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index) } - def potentiality = avg_cost_index + 4099.978248 + def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248 potentiality = DataHandleUtils.getDoubleRound(6, potentiality) def map = [:] - map.id = new ObjectId().toString() + map.id = d._id.toString() map.city = d.city map.region = d.region map.name = d.name @@ -130,7 +134,7 @@ class DistrictIndexBatch extends BaseBatch{ datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] def newUrl = targetUrl +"/"+map.id - // httpSaveData(newUrl, map) + httpSaveData(newUrl, map) println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price } catch (Exception e) { e.printStackTrace() @@ -154,7 +158,8 @@ class DistrictIndexBatch extends BaseBatch{ } def getHousePrice(d, dists, priceMap){ - def areas = findHouse(d, 20200101, 20220101) + //def areas = findHouse(d, 20200101, 20220101) + def areas = findHouseByOracle(d._id.toString()); def avg_area = null if(areas){ avg_area = areas.sum()/areas.size(); @@ -179,11 +184,18 @@ class DistrictIndexBatch extends BaseBatch{ } if(selectDists){ - def allAreas = [] + println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size() +// def allAreas = [] + def ids = [] for(def sd : selectDists){ - def areaDatas = findHouse(sd, 20190101, 20220101) - allAreas.addAll(areaDatas) +// //def areaDatas = findHouse(sd, 20190101, 20220101 +// def areaDatas = findHouseByOracle(sd._id.toString()); +// allAreas.addAll(areaDatas) + ids << sd._id.toString() } + + def idstr = ids.join("','") + def allAreas = findHouseByOracle2(idstr); if(allAreas){ avg_area = allAreas.sum()/allAreas.size(); } @@ -311,6 +323,27 @@ class DistrictIndexBatch extends BaseBatch{ } + + def findHouseByOracle(def distId){ + + def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='440100' AND COMMID='${distId}'"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + allRows << r.BUILDAREA + }) + return allRows + } + + def findHouseByOracle2(def idstr){ + + def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='440100' AND COMMID in ('${idstr}')"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + allRows << r.BUILDAREA + }) + return allRows + } + def findHouse(def dist, def startDate, def endDate){ def nameQuery = getAllName(dist); final StringBuffer sqlBuffer = new StringBuffer() From a7e6880157c53d47de6cb0f3e29e5ad5f7cf125c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 10 Jan 2022 23:58:58 +0800 Subject: [PATCH 160/212] update --- .../com/fangjia/script/gj/level/DistrictIndexBatch.groovy | 3 +++ 1 file changed, 3 insertions(+) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 44fe7b006..dbe384bc1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -193,6 +193,9 @@ class DistrictIndexBatch extends BaseBatch{ // allAreas.addAll(areaDatas) ids << sd._id.toString() } + if(ids.size()>=901){ + ids = ids[0..900] + } def idstr = ids.join("','") def allAreas = findHouseByOracle2(idstr); From 0ff7f23f1eefd4f48aab362eb3cdf0d85bca1776 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 11 Jan 2022 11:56:55 +0800 Subject: [PATCH 161/212] update bug --- .../script/gj/level/DistrictIndexBatch.groovy | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index dbe384bc1..a53edd4d5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -3,6 +3,7 @@ package com.fangjia.script.gj.level import com.fangjia.core.util.HttpRequest import com.fangjia.httpclient.UserAgents import com.fangjia.script.BaseBatch +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.ccb.FileUtils @@ -45,7 +46,7 @@ class DistrictIndexBatch extends BaseBatch{ static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -60,15 +61,17 @@ class DistrictIndexBatch extends BaseBatch{ } def run(){ + def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = ["上海"]//CommonDatautils.findAllCity(); for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) // dists = dists[0..20] - calcute(city, dists) + def code = cityCodeDatas.get(city) + calcute(city, dists, code) } } - def calcute(city, dists){ + def calcute(city, dists, citycode){ def weekend = DateUtils.getFirstDayOfWeek(new Date()); def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) def costDatas = getCostByCity(city) @@ -91,7 +94,7 @@ class DistrictIndexBatch extends BaseBatch{ avg_charge = gtAvgEstateCharge(list) rank = getPriceInRegionRank(d, list, priceMap) } - def total_price = getHousePrice(d, dists, priceMap); + def total_price = getHousePrice(d, dists, priceMap, citycode); def avg_cost = getAvgCost(d, costDatas) def total_price_index = 0 @@ -123,7 +126,7 @@ class DistrictIndexBatch extends BaseBatch{ map.city = d.city map.region = d.region map.name = d.name - map.avg_total_price = total_price + map.avg_total_price = total_price_index map.avg_estate_charge = avg_charge map.price_rank = rank map.avg_cost = avg_cost @@ -157,9 +160,9 @@ class DistrictIndexBatch extends BaseBatch{ // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") } - def getHousePrice(d, dists, priceMap){ + def getHousePrice(d, dists, priceMap, citycode){ //def areas = findHouse(d, 20200101, 20220101) - def areas = findHouseByOracle(d._id.toString()); + def areas = findHouseByOracle(d._id.toString(), citycode); def avg_area = null if(areas){ avg_area = areas.sum()/areas.size(); @@ -198,7 +201,7 @@ class DistrictIndexBatch extends BaseBatch{ } def idstr = ids.join("','") - def allAreas = findHouseByOracle2(idstr); + def allAreas = findHouseByOracle2(idstr,citycode); if(allAreas){ avg_area = allAreas.sum()/allAreas.size(); } @@ -327,9 +330,9 @@ class DistrictIndexBatch extends BaseBatch{ - def findHouseByOracle(def distId){ + def findHouseByOracle(def distId, def citycode){ - def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='440100' AND COMMID='${distId}'"; + def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='${citycode}' AND COMMID='${distId}'"; def allRows = [] sqlInstance.eachRow(sql,{r-> allRows << r.BUILDAREA @@ -337,9 +340,9 @@ class DistrictIndexBatch extends BaseBatch{ return allRows } - def findHouseByOracle2(def idstr){ + def findHouseByOracle2(def idstr, def citycode){ - def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='440100' AND COMMID in ('${idstr}')"; + def sql = "select BUILDAREA from bigdata_forsale_gj where CITYID='${citycode}' AND COMMID in ('${idstr}')"; def allRows = [] sqlInstance.eachRow(sql,{r-> allRows << r.BUILDAREA From 40928486ea2371bba3745ada0ce6bbdc91dffb28 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 14 Jan 2022 10:54:04 +0800 Subject: [PATCH 162/212] update --- .../ccbevaluation/CcbCommonUtils.groovy | 3 +- .../DistrictGuidePriceBatch.groovy | 2 +- .../ccbevaluation/ExportOracleData.groovy | 64 ++-- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../script/ccbevaluation/LoadGuide.groovy | 32 +- .../ccbevaluation/data/ExcelDataTest.groovy | 280 +++++++++++++++--- .../ronghe/ExportDistrictRongHe.groovy | 12 + .../script/fangjiatong/DownPdfCheck.groovy | 29 +- .../fuzhou/DistrictPriceExportBatch.groovy | 28 ++ .../com/fangjia/script/fuzhou/Test.groovy | 90 +++--- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../script/gj/level/DistrictIndexBatch.groovy | 42 ++- .../script/thirdPriceCrawl/CrawalUtil.groovy | 10 +- .../thirdPriceCrawl/DataStandardBatch.groovy | 2 +- .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 73 +++-- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- 17 files changed, 513 insertions(+), 170 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 75b1a0e45..42f56e208 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -82,9 +82,10 @@ class CcbCommonUtils { return rets } - public static def findGuidePrice(city){ + public static def findGuidePrice(city, startDate, endDate){ def q = new BasicDBObject() q.put("city", city) + q.put('c_date', new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD(startDate)).append('$lt', DateUtils.parseFromTightYYYYMMDD(endDate))); def dbcursor = DISTRICT_GUIDE_PRICE_COLL.find(q) def rets = [] for(def d : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index b64fdcd1a..2f73eda31 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findGuidePrice(city) + def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 24dbf7876..736221526 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -24,12 +24,14 @@ class ExportOracleData { } def run() { -// + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // exportGuidedPrice() -// exportCommunityGuidedPrice() + exportCommunityGuidedPrice(lastMonth.toString()+"28") // // //exportThirdPartyPrice()//废弃掉,老表 -// exportThirdPartyPrice2(202111) +// exportThirdPartyPrice2(lastMonth) // // // @@ -40,20 +42,20 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() // exportBigdataDistrictData() - exportBigdataCommunity() +// exportBigdataCommunity() // exportBigdataDeal("2021-11-01") -// exportAvmEvaluationGJNB(202112) -// exportAVMEvaluationItemGJNB(202112) -// exportCommunityBasePrice(202112) +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) +// exportCommunityBasePrice(curMonth) // -// exportCityPrice(202112) -// exportRegionPrice(202112) +// exportCityPrice(curMonth) +// exportRegionPrice(curMonth) -// exportCommunityBasePriceGj(202112) +// exportCommunityBasePriceGj(curMonth) -// exportCommunityPdescribeGj(202112) -// exportCommCaseDetailGj(202112) +// exportCommunityPdescribeGj(curMonth) +// exportCommCaseDetailGj(curMonth) // exportLastFile() } @@ -255,26 +257,26 @@ class ExportOracleData { println "GUIDED_PRICE_GJ 上传Ftp完成......" } - def exportCommunityGuidedPrice() { - def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj"; + def exportCommunityGuidedPrice(def date) { + def sql = "select ID, COMMID, COMMNAME, CITYID, CITYNAME, DISTID, DISTNAME, AVGPRICE, SOURCETYPE, REMARK, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME , SOURCECODE from community_guide_price_gj where createtime>=to_date('${date}','yyyyMMdd')"; def allRows = [] -// sqlInstance.eachRow(sql, { r -> -// def data = [] -// data << r.ID -// data << r.COMMID -// data << r.COMMNAME -// data << r.CITYID -// data << r.CITYNAME -// data << r.DISTID -// data << r.DISTNAME -// data << r.AVGPRICE -// data << r.SOURCETYPE -// data << r.REMARK -// data << r.CREATETIME -// data << r.UPDATETIME -// data << r.SOURCECODE -// allRows << data -// }); + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMID + data << r.COMMNAME + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.AVGPRICE + data << r.SOURCETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM") + "28"; def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_GUIDE_PRICE_GJ_ADD_${curDate}_0001.dat" if (CommonDatautils.isWindow()) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 03893ad8b..ef1f1d1ef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2021-10-1", "2021-11-1") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index c6fa7f82c..84f2232df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -4,6 +4,8 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.fangjia.core.RegionData +import com.fangjia.script.gj.district.SeaweedDataMatchProcess class LoadGuide { public static final DBCollection DISTRICT_GUIDE_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "district_guide_price", "mongo.db.fangjia.com", 28017) @@ -14,13 +16,31 @@ class LoadGuide { } def run(){ +// match("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") + +// saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞指导价218.xlsx") // saveDongGuangPrice("D:\\数据\\建行\\房产估值\\指导价\\东莞二手房政府指导价.xls") // saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx") -// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安成交参考价格.xlsx") + savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") - savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") + + } + def match(def filename){ + def city = "西安" + ExcelUtils.modify(filename,{d-> + def newRegion = RegionData.formatRegion(city, d['A']) + if(newRegion){ + def rMap = SeaweedDataMatchProcess.doMatch(city, newRegion, d['B'], null); + if(rMap){ + d['D'] = city + d['E'] = rMap.region + d['F'] = rMap.name + } + } + },0) } @@ -32,8 +52,8 @@ class LoadGuide { def o = new BasicDBObject() o.city = d['E'] o.region = d['A'] - o.name = d['C'] - o.avg_price = d['D'].toString().toDouble().toInteger() + o.name = d['B'] + o.avg_price = d['C'].toString().toDouble().toInteger() o.match_city = d['E'] o.match_region = d['F'] o.match_name = d['G'] @@ -57,11 +77,11 @@ class LoadGuide { o.city = d['E'] o.region = d['A'] o.name = d['B'] - o.avg_price = d['D'].toString().toDouble().toInteger() + o.avg_price = d['C'].toString().toDouble().toInteger() o.match_city = d['E'] o.match_region = d['F'] o.match_name = d['G'] - o.house_count = d['C'].toString().toDouble().toInteger() + // o.house_count = d['C'].toString().toDouble().toInteger() o.source_type = 1 //1:成交参考价 2:参考价 o.c_date = new Date() rets << o diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index 8373b928c..8ec8ba66a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -4,7 +4,107 @@ import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.GeoHelper +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection + + +DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion && !d['K']) { +// def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']); +// if(!rMap && d['G']){ +// rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['G'], d['H']); +// } +// if(rMap){ +// d['O'] = rMap.city +// d['P'] = rMap.region +// d['Q'] = rMap.name +// d['R'] = rMap.address +// } +// } +//}); + + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion && !d['K']){ +// +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'],"district") +// if(dist){ +// d['K']="完全一致" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'],"district") +// if(dist){ +// d['K']="映射别名" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district") +// if(rets){ +// dist = rets[0] +// } +// if(rets){ +// d['K']="别名" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// } +// } +// } +// } +// +//},0) + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion && !d['K'] && d['G']){ +// +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['G'],"district") +// if(dist){ +// d['K']="完全一致(高德)" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district") +// if(dist){ +// d['K']="映射别名(高德)" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// }else{ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") +// if(rets){ +// dist = rets[0] +// } +// if(rets){ +// d['K']="别名(高德)" +// d['L'] = dist.estate_type2 +// d['M'] = dist.name_cat +// } +// } +// } +// } +// +//},0) + + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\价格过滤小区融合情况_1209.xlsx",{d-> +// def dist = CommonDatautils.findDistrictFieldById(d['F'], ["address","lat","lng"]) +// if(dist){ +// d['Z'] = dist.address +// if(dist.lat){ +// d['AA'] = dist.lat/10000000000.0 +// d['AB'] = dist.lng/10000000000.0 +// } +// } +//},0) //def cityDatas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\小区\\重庆高德反查数据_1529.xlsx",0,1) //def cityDataMap = [:] @@ -74,54 +174,156 @@ def city = " //},0) -//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例(合并).xlsx",{d-> // def obj = CommonDatautils.findDistrict("广州", d['C'], d['D'], "district") // if(obj){ -// def year = CommonUtils.getDistrictAge(obj) -// if(year){ -// d['H'] = year -// } -// if(obj.alias){ -// d['K'] = obj.alias.join(" ") +//// def year = CommonUtils.getDistrictAge(obj) +//// if(year){ +//// d['H'] = year +//// } +//// if(obj.alias){ +//// d['K'] = obj.alias.join(" ") +//// } +// if(obj.mapping_alias){ +// d['X'] = obj.mapping_alias.join(" ") // } // // } //},0) -def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1) -def scoreMap = [:] -for(def d : data1s){ - scoreMap.put(d['B'], d) -} +//def data1s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",0, 1) +//def scoreMap = [:] +//for(def d : data1s){ +// scoreMap.put(d['B'], d) +//} +// +//def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1) +//def levelMap = [:] +//for(def d : data2s){ +// levelMap.put(d['E'], d) +//} +// +//ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> +// if(d['E']){ +// def row = scoreMap.get(d['E']) +// if(row){ +// d['I'] = row['G'] +// d['J'] = row['H'] +// d['K'] = row['I'] +// d['L'] = row['J'] +// d['M'] = row['K'] +// d['N'] = row['L'] +// d['O'] = row['M'] +// } +// +// def data = levelMap.get(d['E']) +// if(data){ +// d['P'] = data['G'] +// d['Q'] = data['H'] +// d['R'] = data['I'] +// d['S'] = data['J'] +// d['T'] = data['K'] +// d['U'] = data['L'] +// d['V'] = data['M'] +// } +// } +//},0) -def data2s = ExcelUtils.read("D:\\数据\\广发\\图片转换汇总1.8.xlsx",1, 1) -def levelMap = [:] -for(def d : data2s){ - levelMap.put(d['E'], d) -} +//ExcelUtils.modify("D:\\数据\\成交\\福州成交.xlsx",{d-> +// def o = new BasicDBObject() +// o.area = d['D'].toDouble() +// o.avg_price = d['F'].toInteger() +// o.city = "福州" +// o.district_name = d['C'] +// o.region = d['B'] +// o.source = "贝壳找房" +// o.trade_date = DateUtils.parseFromTightYYYYMMDD(d['M']) +// o.total_price = d['E'].toInteger()*10000 +// if(d['I']){ +// o.hall = d['I'].toInteger() +// } +// +// o.m_date = new Date() +// if(d['H']){ +// o.room = d['H'].toInteger() +// } +// +// o.create_date = new Date() +// o.status = 2 +// DEAL_PRICE.save(o) +// +//},0) -ExcelUtils.modify("D:\\数据\\广发\\楼盘评级测试案例.xlsx",{d-> - if(d['E']){ - def row = scoreMap.get(d['E']) - if(row){ - d['I'] = row['G'] - d['J'] = row['H'] - d['K'] = row['I'] - d['L'] = row['J'] - d['M'] = row['K'] - d['N'] = row['L'] - d['O'] = row['M'] - } - def data = levelMap.get(d['E']) - if(data){ - d['P'] = data['G'] - d['Q'] = data['H'] - d['R'] = data['I'] - d['S'] = data['J'] - d['T'] = data['K'] - d['U'] = data['L'] - d['V'] = data['M'] +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// if(d['K'] && d['K']=="别名"){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'],"district") +// def dist = null +// if(rets){ +// dist = rets[0] +// } +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['C'])){ +// alias << d['C'] +// } +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// } +// } +//}); + + +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// if(d['K'] && d['K']=="映射别名(高德)" && d['G']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['G'],"district") +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['C'])){ +// alias << d['C'] +// } +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// } +// } +//}); + + +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> + if(d['K'] && d['K']=="别名(高德)" && d['G']){ + def newRegion = RegionData.formatRegion(d['A'], d['B']) + if(newRegion){ + def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") + def dist = null + if(rets){ + dist = rets[0] + } + if(dist){ + def alias = dist.mapping_alias + if(!alias){ + alias = [] + } + if(!alias.contains(d['C'])){ + alias << d['C'] + } + def udata = new BasicDBObject() + udata.put("mapping_alias",alias) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + } } } -},0) +}); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index c7847b806..683312af1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -27,6 +27,7 @@ class ExportDistrictRongHe extends BaseBatch{ def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + //检查ok文件是否上传 // exportAllBigdataDealFJW() for(def city : citys) { def cityCode = cityCodeDataMap.get(city) @@ -47,6 +48,7 @@ class ExportDistrictRongHe extends BaseBatch{ exportBigdataCommlayoutpicFjw(cityCode) } exportEvaluateRate() + exportOk() } @@ -593,4 +595,14 @@ class ExportDistrictRongHe extends BaseBatch{ println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } + def exportOk(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy index bd195836d..2e5201a8c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fangjiatong/DownPdfCheck.groovy @@ -13,7 +13,34 @@ class DownPdfCheck { static main(args) { def task = new DownPdfCheck(); - task.run(); +// task.run(); + + task.czLogRun() + } + + def czLogRun(){ + def files = getAllFiles("D:\\数据\\常州") + if(files){ + for(def filepath : files){ + read(filepath); + } + } + + } + + + def read(file){ + // File file = new File(fileName); + def index = 0 + + file.eachLine{it-> + if(it.contains("4ead0347b4055be6989e726e")||it.contains("61a2e0")){ + index++ + + println it + } + } + } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy index 87a2d9962..e8cd3476a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/DistrictPriceExportBatch.groovy @@ -8,6 +8,7 @@ import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.jusfoun.JusfounCommon import com.fangjia.script.util.DateUtils import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.io.FileUtils import java.util.concurrent.TimeUnit @@ -79,6 +80,14 @@ class DistrictPriceExportBatch extends BaseBatch{ def lastDate = new Date() def startDate = DateUtils.getFirstDayOfMonth(lastDate, -1) +// List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) +// def map2 = [:] +// for(def s : list2) { +// def row = s.split("@\\|@") +// def key = row[2] + "##" + row[3] + "##" + row[4] +// map2.put(key, row[5]) +// } + def result = [] for(def rMap : districts){ def dbo = CommonDatautils.queryDistrictPriceByDate3(city, rMap.region, rMap.name, startDate, lastDate) @@ -90,6 +99,25 @@ class DistrictPriceExportBatch extends BaseBatch{ m << rMap.region m << rMap.name m << dbo.get("avg_price") +// def lastMonthPrice = map2.get(rMap.city + "##" + rMap.region+"##"+rMap.name) +// if(lastMonthPrice && dbo.get("avg_price")){ +// lastMonthPrice = lastMonthPrice.toInteger() +// def markup = (dbo.get("avg_price")-lastMonthPrice)*100.0/lastMonthPrice +// if(Math.abs(markup)>10){ +// def random = 0 +// if(dbo.get("avg_price")-lastMonthPrice>=0){ +// random = com.fangjia.core.util.RandomUtil.random(1, 9) +// }else{ +// random = com.fangjia.core.util.RandomUtil.random(-9, 0) +// } +// def newPrice = lastMonthPrice*random/100 + lastMonthPrice +// m << newPrice.toInteger() +// }else{ +// m << dbo.get("avg_price") +// } +// }else{ +// m << dbo.get("avg_price") +// } m << curDate.toInteger(); result << m } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy index 88100888a..0231b21c0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test.groovy @@ -48,51 +48,51 @@ import org.bson.types.ObjectId // return location; //} -//List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202109.csv")) -//def map1 = [:] -//for(def s : list1){ -// def row = s.split("@\\|@") -// def key = row[2]+"##"+row[3]+"##"+row[4] -// map1.put(key, row[5]) -//} -// -//List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202108.csv")) -//def map2 = [:] -//for(def s : list2) { -// def row = s.split("@\\|@") -// def key = row[2] + "##" + row[3] + "##" + row[4] -// map2.put(key, row[5]) -//} -//def rows = [] -//map1.each {k,v-> -// def row = [] -// def keys = k.split("##") -// def lastPrice = map2.get(k) -// rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] -//} -//ExcelUtils.write("D:\\数据\\福州\\价格比对.xlsx", rows, "数据") -ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d-> - def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) - if(obj){ - if(obj.value_type == 1){ - d["G"] = "指导价" - }else if(obj.value_type == 2){ - d["G"] ="自身样本" - }else if(obj.value_type == 3){ - d["G"] = "第三方" - }else if(obj.value_type == 4){ - d["G"] = "交易数据" - }else if(obj.value_type == 5){ - d["G"] ="周边楼盘" - }else if(obj.value_type == 6){ - d["G"] ="推导" - }else if(obj.value_type == 9){ - d["G"] = "神经网络" - }else if(obj.value_type == 10){ - d["G"] = "无方式" - } - } -},0) +List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202112.csv")) +def map1 = [:] +for(def s : list1){ + def row = s.split("@\\|@") + def key = row[2]+"##"+row[3]+"##"+row[4] + map1.put(key, row[5]) +} + +List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202111.csv")) +def map2 = [:] +for(def s : list2) { + def row = s.split("@\\|@") + def key = row[2] + "##" + row[3] + "##" + row[4] + map2.put(key, row[5]) +} +def rows = [] +map1.each {k,v-> + def row = [] + def keys = k.split("##") + def lastPrice = map2.get(k) + rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString()] +} +ExcelUtils.write("D:\\数据\\福州\\价格比对12-2.xlsx", rows, "数据") +//ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州涨幅在10%以上小区.xlsx",{d-> +// def obj = CommonDatautils.queryDistrictPrice(d['A'],d['B'], d['C'], DateUtils.parseFromTightByPattern("2021-10-10","yyyy-MM-dd")) +// if(obj){ +// if(obj.value_type == 1){ +// d["G"] = "指导价" +// }else if(obj.value_type == 2){ +// d["G"] ="自身样本" +// }else if(obj.value_type == 3){ +// d["G"] = "第三方" +// }else if(obj.value_type == 4){ +// d["G"] = "交易数据" +// }else if(obj.value_type == 5){ +// d["G"] ="周边楼盘" +// }else if(obj.value_type == 6){ +// d["G"] ="推导" +// }else if(obj.value_type == 9){ +// d["G"] = "神经网络" +// }else if(obj.value_type == 10){ +// d["G"] = "无方式" +// } +// } +//},0) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 9968280b4..2f94c98c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202111.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index a53edd4d5..f2fbed25d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -62,7 +62,8 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() - def citys = ["上海"]//CommonDatautils.findAllCity(); + def citys = CommonDatautils.findAllCity(); + citys.remove("上海") for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) // dists = dists[0..20] @@ -156,13 +157,19 @@ class DistrictIndexBatch extends BaseBatch{ fileName = "D:\\数据\\评级\\上海综合潜力.dat" } - FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", true); // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") } def getHousePrice(d, dists, priceMap, citycode){ //def areas = findHouse(d, 20200101, 20220101) - def areas = findHouseByOracle(d._id.toString(), citycode); + + def areas = [] + if(citycode){ + areas = findHouseByOracle(d._id.toString(), citycode); + }else{ + areas = findHouse(d, 20200101, 20220101) + } def avg_area = null if(areas){ avg_area = areas.sum()/areas.size(); @@ -188,24 +195,27 @@ class DistrictIndexBatch extends BaseBatch{ if(selectDists){ println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size() -// def allAreas = [] + def allAreas = [] def ids = [] - for(def sd : selectDists){ -// //def areaDatas = findHouse(sd, 20190101, 20220101 -// def areaDatas = findHouseByOracle(sd._id.toString()); -// allAreas.addAll(areaDatas) - ids << sd._id.toString() - } - if(ids.size()>=901){ - ids = ids[0..900] - } + if(citycode){ + for(def sd : selectDists){ + ids << sd._id.toString() + } + if(ids.size()>=901){ + ids = ids[0..900] + } - def idstr = ids.join("','") - def allAreas = findHouseByOracle2(idstr,citycode); + def idstr = ids.join("','") + allAreas = findHouseByOracle2(idstr,citycode); + }else{ + for(def sd : selectDists){ + def areaDatas = findHouse(sd, 20190101, 20220101) + allAreas.addAll(areaDatas) + } + } if(allAreas){ avg_area = allAreas.sum()/allAreas.size(); } - } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy index a284b5c6f..d283bbf5c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/CrawalUtil.groovy @@ -98,9 +98,9 @@ class CrawalUtil { } - static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null) { + static def getDocumentByABuYun(def url, def errorMsg, def charset, def language = null, def isJson=false) { def flag = true; - Document document = null; + def document = null; def count = 0; while (flag) { def content = null; @@ -121,7 +121,11 @@ class CrawalUtil { } } if (content) { - document = Jsoup.parse(content); + if(!isJson){ + document = Jsoup.parse(content); + }else{ + document = content + } } if (!content || (content && errorMsg && content.contains(errorMsg))) { count++; diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy index 6dedf2c7b..1d80f9b2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStandardBatch.groovy @@ -364,7 +364,7 @@ class DataStandardBatch extends BaseBatch{ def standardDistrict(def data, def avg_price){ //名称匹配,名称别名匹配,地址别名匹配 //def district = getMatchName(data.city, data.region, data.name, data.address,"district") - data.name = data.name.replace("(","").replace(")","") + data.name = data.name.replace("(","").replace(")","").replace("数据由万科物业提供","").trim() def district = null; def type = 0; if(data.name.contains("写字楼")||data.name.contains("办公楼")||data.name.contains("商业")){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 6566d5a16..5d972bd90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "城市房产"// - def citys = findCrawlCity(source, 202110); + def source = "58同城"// + def citys = findCrawlCity(source, 202111); for(def city : citys){ - def count1 = findCount(source, city, 202110); - def count2 = findCount(source, city, 202111); + def count1 = findCount(source, city, 202111); + def count2 = findCount(source, city, 202112); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index ae2639418..10c39958e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -12,6 +12,7 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import net.sf.json.JSONObject import org.jsoup.nodes.Element import org.jsoup.select.Elements @@ -35,17 +36,16 @@ class Esf58Crawl extends BaseBatch{ def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def static final distMap = [:] - def static useDistKeys = [] + def static useDistKeys = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","不","了","这","中","大","有","和","人","是","在","一","的","为","上","个","国","我","以","要","他","时","来","用","们","生","到","作","地","于","出","就","分","对","成","会","可","主","发","年","动","同","工","也","能","下","过","子","说","产","种","面","而","方","后","多","定","行","学","法","所","民","得","经","十","三","之","进","着","等","部","度","家","电","力","里","如","水","化","高","自","二","理","起","小","物","现","实","加","量","都","两","体","制","机","当","使","点","从","业","本","去","把","性","好","应","开","它","合","还","因","由","其","些","然","前","外","天","政","四","日","那","社","义","事","平","形","相","全","表","间","样","与","关","各","重","新","线","内","数","正","心","反","你","明","看","原","又","么","利","比","或","但","质","气","第","向","道","命","此","变","条","只","没","结","解","问","意","建","月","公","无","系","军","很","情","者","最","立","代","想","已","通","并","提","直","题","党","程","展","五","果","料","象","员","革","位","入","常","文","总","次","品","式","活","设","及","管","特","件","长","求","老","头","基","资","边","流","路","级","少","图","山","统","接","知","较","长","将","组","见","计","别","她","手","角","期","根","论","运","农","指","几","九","区","强","放","决","西","被","干","做","必","战","先","回","则","任","取","据","处","队","南","给","色","光","门","即","保","治","北","造","百","规","热","领","七","海","地","口","东","导","器","压","志","世","金","增","争","济","阶","油","思","术","极","交","受","联","什","认","六共","权","收","证","改","清","已","美","再","采","转","更","单","风","切","打","白","教","速","花","带","安","场","身","车","例","真","务","具","万","每","目","至","达","走","积","示","议","声","报","斗","完","类","八","离","华","名","确","才","科","张","信","马","节","话","米","整","空","元","况","今","集","温","传","土","许","步","群","广","石","记","需","段","研","界","拉","林","律","叫","且","究","观","越","织","装","影","算","低","持","音","众","书","布","复","容","儿","须","际","商","非","验","连","断","深","难","近","矿","千","周","委","素","技","备","半","办","青","省","列","习","响","约","支","般","史","感","劳","便","团","往","酸","历","市","克","何","除","消","构","府","称","太","准","精","值","号","率","族","维","划","选","标","写","存","候","毛","亲","快","效","斯","院","查","江","型","眼","王","按","格","养","易","置","派","层","片","始","却","专","状","育","厂","京","识","适","属","圆","包","火","住","调","满","县","局","照","参","红","细","引","听","该","铁","价","严","首","底","液","官","德","调","随","病","苏","失","尔","死","讲","配","女","黄","推","显","谈","罪","神","艺","呢","席","含","企","望","密","批","营","项","防","举","球","英","氧","势","告","李","台","落","木","帮","轮","破","亚","师","围","注","远","字","材","排","供","河","态","封","另","施","减","树","溶","怎","止","案","言","士","均","武","固","叶","鱼","波","视","仅","费","紧","爱","左","章","早","朝","害","续","轻","服","试","食","充","兵","源","判","护","司","足","某","练","差","致","板","田","降","黑","犯","负","击","范","继","兴","似","余","坚","曲","输","修","的","故","城","夫","够","送","笔","船","占","右","财","吃","富","春","职","觉","汉","画","功","巴","跟","虽","杂","飞","检","吸","助","升","阳","互","初","创","抗","考","投","坏","策","古","径","换","未","跑","留","钢","曾","端","责","站","简","述","钱","副","尽","帝","射","草","冲","承","独","令","限","阿","宣","环","双","请","超","微","让","控","州","良","轴","找","否","纪","益","依","优","顶","础","载","倒","房","突","坐","粉","敌","略","客","袁","冷","胜","绝","析","块","剂","测","丝","协","重","诉","念","陈","仍","罗","盐","友","洋","错","苦","夜","刑","移","频","逐","靠","混","母","短","皮","终","聚","汽","村","云","哪","既","距","卫","停","烈","央","察","烧","行","迅","境","若","印","洲","刻","括","激","孔","搞","甚","室","待","核","校","散","侵","吧","甲","游","久","菜","味","旧","模","湖","货","损","预","阻","毫","普","稳","乙","妈","植","息","扩","银","语","挥","酒","守","拿","序","纸","医","缺","雨","吗","针","刘","啊","急","唱","误","训","愿","审","附","获","茶","鲜","粮","斤","孩","脱","硫","肥","善","龙","演","父","渐","血","欢","械","掌","歌","沙","著","刚","攻","谓","盾","讨","晚","粒","乱","燃","矛","乎","杀","药","宁","鲁","贵","钟","煤","读","班","伯","香","介","迫","句","丰","培","握","兰","担","弦","蛋","沉","假","穿","执","答","乐","谁","顺","烟","缩","征","脸","喜","松","脚","困","异","免","背","星","福","买","染","井","概","慢","怕","磁","倍","祖","皇","促","静","补","评","翻","肉","践","尼","衣","宽","扬","棉","希","伤","操","垂","秋","宜","氢","套","笔","督","振","架","亮","末","宪","庆","编","牛","触","映","雷","销","诗","座","居","抓","裂","胞","呼","娘","景","威","绿","晶","厚","盟","衡","鸡","孙","延","危","胶","还","屋","乡","临","陆","顾","掉","呀","灯","岁","措","束","耐","剧","玉","赵","跳","哥","季","课","凯","胡","额","款","绍","卷","齐","伟","蒸","殖","永","宗","苗","川","炉","岩","弱","零","杨","奏","沿","露","杆","探","滑","镇","饭","浓","航","怀","赶","库","夺","伊","灵","税","了","途","灭","赛","归","召","鼓","播","盘","裁","险","康","唯","录","菌","纯","借","糖","盖","横","符","私","努","堂","域","枪","润","幅","哈","竟","熟","虫","泽","脑","壤","碳","欧","遍","侧","寨","敢","彻","虑","斜","薄","庭","都","纳","弹","饲","伸","折","麦","湿","暗","荷","瓦","塞","床","筑","恶","户","访","塔","奇","透","梁","刀","旋","迹","卡","氯","遇","份","毒","泥","退","洗","摆","灰","彩","卖","耗","夏","择","忙","铜","献","硬","予","繁","圈","雪","函","亦","抽","篇","阵","阴","丁","尺","追","堆","雄","迎","泛","爸","楼","避","谋","吨","野","猪","旗","累","偏","典","馆","索","秦","脂","潮","爷","豆","忽","托","惊","塑","遗","愈","朱","替","纤","粗","倾","尚","痛","楚","谢","奋","购","磨","君","池","旁","碎","骨","监","捕","弟","暴","割","贯","殊","释","词","亡","壁","顿","宝","午","尘","闻","揭","炮","残","冬","桥","妇","警","综","招","吴","付","浮","遭","徐","您","摇","谷","赞","箱","隔","订","男","吹","乐","园","纷","唐","败","宋","玻","巨","耕","坦","荣","闭","湾","键","凡","驻","锅","救","恩","剥","凝","碱","齿","截","炼","麻","纺","禁","废","盛","版","缓","净","睛","昌","婚","涉","筒","嘴","插","岸","朗","庄","街","藏","姑","贸","腐","奴","啦","惯","乘","伙","恢","匀","纱","扎","辩","耳","彪","臣","亿","璃","抵","脉","秀","萨","俄","网","舞","店","喷","纵","寸","汗","挂","洪","着","贺","闪","柬","爆","烯","津","稻","墙","软","勇","像","滚","厘","蒙","芳","肯","坡","柱","荡","腿","仪","旅","尾","轧","冰","贡","登","黎","削","钻","勒","逃","障","氨","郭","峰","币","港","伏","轨","亩","毕","擦","莫","刺","浪","秘","援","株","健","售","股","岛","甘","泡","睡","童","铸","汤","阀","休","汇","舍","牧","绕","炸","哲","磷","绩","朋","淡","尖","启","陷","柴","呈","徒","颜","泪","稍","忘","泵","蓝","拖","洞","授","镜","辛","壮","锋","贫","虚","弯","摩","泰","幼","廷","尊","窗","纲","弄","隶","疑","氏","宫","姐","震","瑞","怪","尤","琴","循","描","膜","违","夹","腰","缘","珠","穷","森","枝","竹","沟","催","绳","忆","邦","剩","幸","浆","栏","拥","牙","贮","礼","滤","钠","纹","弹","罢","拍","咱","喊","袖","埃","勤","罚","焦","潜","伍","墨","欲","缝","姓","刊","饱","仿","奖","铝","鬼","丽","跨","默","挖","链","扫","喝","袋","炭","污","幕","诸","弧","励","梅","奶","洁","灾","舟","鉴","苯","讼","抱","毁","率","懂","寒","智","埔","寄","届","跃","渡","挑","丹","艰","贝","碰","拔","爹","戴","码","梦","芽","熔","赤","渔","哭","敬","颗","奔","藏","铅","熟","仲","虎","稀","妹","乏","珍","申","桌","遵","允","隆","螺","仓","魏","锐","晓","氮","兼","隐","碍","赫","拨","忠","肃","缸","牵","抢","博","巧","壳","兄","杜","讯","诚","碧","祥","柯","页","巡","矩","悲","灌","龄","伦","票","寻","桂","铺","圣","恐","恰","郑","趣","抬","荒","腾","贴","柔","滴","猛","阔","辆","妻","填","撤","储","签","闹","扰","紫","砂","递","戏","吊","陶","伐","喂","疗","瓶","婆","抚","臂","摸","忍","虾","蜡","邻","胸","巩","挤","偶","弃","槽","劲","乳","邓","吉","仁","烂","砖","租","乌","舰","伴","瓜","浅","丙","暂","燥","橡","柳","迷","暖","牌","纤","秧","胆","详","簧","踏","瓷","谱","呆","宾","糊","洛","辉","愤","竞","隙","怒","粘","乃","绪","肩","籍","敏","涂","熙","皆","侦","悬","掘","享","纠","醒","狂","锁","淀","恨","牲","霸","爬","赏","逆","玩","陵","祝","秒","浙","貌","役","彼","悉","鸭","着","趋","凤","晨","畜","辈","秩","卵","署","梯","炎","滩","棋","驱","筛","峡","冒","啥","寿","译","浸","泉","帽","迟","硅","疆","贷","漏","稿","冠","嫩","胁","芯","牢","叛","蚀","奥","鸣","岭","羊","凭","串","塘","绘","酵","融","盆","锡","庙","筹","冻","辅","摄","袭","筋","拒","僚","旱","钾","鸟","漆","沈","眉","疏","添","棒","穗","硝","韩","逼","扭","侨","凉","挺","碗","栽","炒","杯","患","馏","劝","豪","辽","勃","鸿","旦","吏","拜","狗","埋","辊","掩","饮","搬","骂","辞","勾","扣","估","蒋","绒","雾","丈","朵","姆","拟","宇","辑","陕","雕","偿","蓄","崇","剪","倡","厅","咬","驶","薯","刷","斥","番","赋","奉","佛","浇","漫","曼","扇","钙","桃","扶","仔","返","俗","亏","腔","鞋","棱","覆","框","悄","叔","撞","骗","勘","旺","沸","孤","粘","吐","孟","渠","屈","疾","妙","惜","仰","狠","胀","谐","抛","霉","桑","岗","嘛","衰","盗","渗","脏","赖","涌","甜","曹","阅","肌","哩","厉","烃","纬","毅","昨","伪","症","煮","叹","钉","搭","茎","笼","酷","偷","弓","锥","恒","杰","坑","鼻","翼","纶","叙","狱","逮","罐","络","棚","抑","膨","蔬","寺","骤","穆","冶","枯","册","尸","凸","绅","坯","牺","焰","轰","欣","晋","瘦","御","锭","锦","丧","旬","锻","垄","搜","佛","扑","邀","亭","酯","迈","舒","脆","酶","闲","忧","酚","顽","羽","涨","卸","仗","陪","薄","辟","惩","杭","姚","肚","捉","飘","漂","昆","欺","吾","郎","烷","汁","呵","饰","萧","雅","邮","迁","燕","撒","姻","赴","宴","烦","削","债","帐","斑","铃","旨","醇","董","饼","雏","姿","拌","傅","腹","妥","揉","贤","拆","歪","葡","胺","丢","浩","徽","昂","垫","挡","览","贪","慰","缴","汪","慌","冯","诺","姜","谊","凶","劣","诬","耀","昏","躺","盈","骑","乔","溪","丛","卢","抹","易","闷","咨","刮","驾","缆","悟","摘","铒","掷","颇","幻","柄","惠","惨","佳","仇","腊","窝","涤","剑","瞧","堡","泼","葱","罩","霍","捞","胎","苍","滨","俩","捅","湘","砍","霞","邵","萄","疯","淮","遂","熊","粪","烘","宿","档","戈","驳","嫂","裕","徙","箭","捐","肠","撑","晒","辨","殿","莲","摊","搅","酱","屏","疫","哀","蔡","堵","沫","皱","畅","叠","阁","莱","敲","辖","钩","痕","坝","巷","饿","祸","丘","玄","溜","曰","逻","彭","尝","卿","妨","艇","吞","韦","怨","矮","歇","郊","禄","捻","漠","粹","颠","宏","冤","肪","饥","呵","仙","押","挨","醛","娃","拾","没","佩","勿","吓","讹","侯","恋","夕","锌","篡","戚","淋","蓬","岂","釉","兆","泊","魂","拘","亡","杠","摧","氟","颂","浑","凌","铀","诱","犁","谴","颁","舶","扯","嘉","萌","犹","滋","焊","舌","匹","媳","肺","掠","酿","烹","疲","驰","鸦","窄","辱","狭","朴","遣","菲","奸","韧","辣","拳","秆","卧","醉","竭","茅","墓","矣","哎","艳","敦","舆","缔","雇","尿","葬","履","契","禽","渣","衬","躲","赔","咸","溉","贼","醋","堤","抖","妃","裤","廉","晴","挽","掀","茫","丑","亥","拦","悠","阐","慧","佐","奇","竖","孝","柜","麟","绣","遥","逝","愁","肖","昭","芬","逢","窑","捷","圜","盲","闸","宙","辐","披","账","狼","幽","绸","蜂","慎","餐","酬","誓","惟","叉","弥","址","帜","芝","砌","唉","仆","涛","臭","翠","盒","劫","慨","炳","阖","寂","椒","倘","拓","畏","喉","巾","颈","垦","拚","兽","蔽","芦","乾","爽","窃","谭","挣","崩","模","褐","传","翅","儒","伞","晃","谬","胚","剖","凑","眠","浊","霜","礁","蔑","抄","闯","洒","碑","蓉","耶","猜","蹲","壶","唤","澳","锯","郡","玲","绵","纽","梳","掏","吁","锤","鼠","穴","椅","殷","遮","吵","萍","厌","畜","俱","夸","吕","囊","捧","雌","闽","饶","瞬","郁","哨","凿","朝","俺","浒","茂","肝","勋","盯","籽","耻","菊","滥","稼","戒","奈","帅","鞭","蚕","镁","询","跌","烤","坛","宅","笛","鄂","蛮","颤","棍","睁","鼎","岌","降","侍","藩","嚷","匪","岳","糟","缠","迪","泄","卑","氛","堪","萝","盛","碘","缚","悦","澄","甫","攀","屠","溢","拱","晰","携","朽","吟","菱","谦","凹","俊","芒","盼","婶","艘","酰","趁","唇","挫","羞","浴","疼","萎","肴","愚","肿","刨","绞","枢","嫁","慕","舱","铲","苹","豫","谕","迭","潘","顷","翁","榜","匠","欠","茬","畴","胃","沾","踪","弊","哼","鹏","歧","桐","沃","悼","惑","溃","蔗","荐","潭","孢","露","诊","庸","聪","嫌","厨","庞","祁","钳","肆","梭","赠","崖","篮","颖","甸","藻","捣","且","撕","诏","贞","赐","慈","炕","胖","兹","差","琼","锈","汛","卓","棵","馈","挠","灶","婴","蒂","肤","衫","沥","仑","勉","沪","逸","蜜","浦","嗓","晕","膏","祭","赢","艾","扮","鹅","怜","蒲","兔","孕","呖","蘖","挪","淑","谣","惧","廊","缅","俘","骄","膀","陡","宰","诞","峻","恼","腺","猎","涡","夷","愉","魔","铵","葛","贾","似","荫","哟","脊","钞","苛","锰","椭","镶","杏","溴","倚","滞","会","氓","捏","斩","傲","匆","僵","卤","烫","衍","榨","拢","裸","屑","咽","坊","舅","渴","翔","邪","拄","窖","猫","砌","钦","媒","脾","勺","柏","栅","噪","昼","耿","扁","辰","秤","得","贩","糕","梁","昙","衷","宦","扔","哇","诈","嘱","藤","卜","冈","悔","廓","皂","拐","氰","杉","玛","矢","寓","瓣","罕","垮","笋","淘","衔","称","恭","喇","帕","桉","秉","帘","铭","蛇","摔","斋","叭","帆","裸","俭","瘤","篷","砸","肢","辟","脖","瞪","暑","卜","竿","歼","笙","酮","蕴","哗","瞎","喀","刃","楔","喘","枚","嵌","挝","厢","粤","甩","拴","膝","恳","腕","娓","熄","锚","忌","愧","哦","荆","圃","骚","丸","蒜","毯","弗","俯","鹿","梢","屯","衙","轿","贱","垒","谅","踢","哑","滔","渥","饷","泳","棕","熬","搁","腈","梨","吻","樱","奠","捆","姨","柏","聘","惕","郓","绑","冀","裹","酥","寡","彦","稠","啡","钝","汝","擅","汰","鳙","埔","敞","嘿","逊","栋","谨","咖","鲤","雀","佣","庵","葫","贿","鳞","拼","搏","谎","塌","忉","腻","戊","怖","坟","禾","刹","嘻","桔","坎","拇","煽","狮","痒","曾","梗","寇","鹰","烛","哄","莽","雯","胳","龟","亟","糠","泌","坪","傻","什","喻","渊","蚌","跪","巷","涅","钊","譬","蕊","膛","侮","奕","枕","辫","况","扼","郝","寥","凄","厦","腥","钧","耦","蹄","戥","屁","诵","匈","桩","钓","涵","倦","袍","抒","屿","蹈","忿","敷","虹","聊","嗣","尉","灿","糙","蹬","嗯","姬","狡","笨","辜","僧","茨","讽","翰","枉","岐","枣","崭","焚","咕","猴","揽","涝","耍","趟","汹","咋","傍","镀","给","爵","虏","劈","璋","踩","瞅","迄","昔","汞","呱","诡","魄","祺","嘲","惶","赃","癌","咐","歉","扳","鄙","庐","聂","便","芡","躯","贬","煌","拧","隋","襄","淤","宠","炊","滇","謇","懒","栓","佑","憾","骆","裙","猖","兜","孵","痼","盥","曝","泣","絮","韵","眷","旷","噢","参","栖","盏","鳌","溅","煎","校","榴","暮","琪","淆","陛","巢","哒","吼","槐","唧","其","沛","乞","蜀","蜇","赚","捍","铰","幂","尧","咒","耽","叮","褂","焕","煞","雹","搓","釜","铬","拣","募","淹","瑰","鲢","茄","灼","邹","躬","觉","娇","焉","彰","鹤","琳","沦","畔","惹","庶","毙","皖","邢","禹","渍","绷","窜","翘","淫","箪","陌","膊","鞑","咳","玫","巫","拂","蕉","澜","赎","绥","锄","囱","赌","颊","缕","寅","躁","稚","庚","苟","氦","魁","珊","蜕","蛭","酌","逗","闺","蔓","撇","豌","朕","缉","襟","镍","桅","荧","侄","卒","佃","瞿","娶","饪","耸","乍","靶","痴","靖","扛","筐","韶","嚣","崔","蓿","岔","氘","娥","剿","霖","喃","搪","雍","裳","撰","豹","骏","慷"] + static def totalCount = new AtomicLong() - //"福州":"fz|304","合肥":"hf|837","芜湖":"wuhu|2045","蚌埠":"bengbu|3470","阜阳":"fy|2325","淮南":"hn|2319","成都":"cd|102", -// "青岛":"qd|122", -// "重庆":"cq|37","石家庄":"sjz|241","广州":"gz|3","深圳":"sz|4","东莞":"dg|413","佛山":"fs|222","杭州":"hz|79","武汉":"wh|158", -// "上海":"sh|2", - def cityList = [ "北京":"bj|1","天津":"tj|18","南京":"nj|172", - "嘉兴":"jx|497","大连":"dl|147","乌鲁木齐":"xj|984","太原":"ty|740","兰州":"lz|952","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362","常州":"cz|463", - "南通":"nt|394","吉林":"jl|700","中山":"zs|771","台州":"tz|403","东营":"dy|623","三亚":"sanya|2422", - "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", - "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", + //"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", +// "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", +// "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", +// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","三亚":"sanya|2422","阜阳":"fy|2325", +// "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", +// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385", + def cityList = ["北京":"bj|1","上海":"sh|2","深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", "淮北":"huaibei|9357","滁州":"chuzhou|10266", "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", @@ -199,7 +199,7 @@ class Esf58Crawl extends BaseBatch{ distMap.put(city, districts) addCrawlKeys(city); - addDetailContent(city, url); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); // Thread.sleep(1000*60*10) } @@ -216,13 +216,13 @@ class Esf58Crawl extends BaseBatch{ def districtName = districts[i].name println "${city}【${i}/${districts.size()}】 "+districtName for(def j=1;j<=districtName.length()-1;j++) { - if(j<=2){ +// if(j<=3){ def keyword = districtName.toString().substring(0, j); - def keyStr = city+"##"+keyword; + def keyStr = keyword; if(!useDistKeys.contains(keyStr)){ useDistKeys << keyStr } - } +// } } } useDistKeys.sort {a,b-> @@ -235,16 +235,17 @@ class Esf58Crawl extends BaseBatch{ - def addDetailContent(def city, def url){ + def addDetailContent(def city, def url, def cityid){ final CountDownLatch latch = new CountDownLatch(useDistKeys.size()) for(def k=0;k0){ + def datas = jsonObj.data + for(def d : datas){ + def obj = findByParams(city, d.areaName, d.name); + def price = null + if(d.price){ + price = d.price.toInteger() + } + if(obj){ + DBObject udata2 = new BasicDBObject() + udata2.put("avg_price", price); + udata2.put("c_date", new Date()); + println "[update] "+city + " "+ d.name + thirdPartyColl.update(new BasicDBObject("_id", obj._id), new BasicDBObject('$set', udata2)); + }else{ + + println "[save] "+city + " "+ d.name + saveThirdParty(["url":d.listName,"city":city,"region":d.areaName,"block":null,"name":d.name,"address": d.address, "avg_price": price]); + } + } + } + }else{ + println "【"+keyword +"】 无结果" + } + + } + } + def addSearchContent(city, url, keyword){ def flag = true; @@ -416,7 +453,7 @@ class Esf58Crawl extends BaseBatch{ obj.put("s_date", SHORT_DATE); obj.put("source", SOURCE); obj.put("addres", metadata.address); - obj.put("avg_price", metadata.avg_price.toInteger()); + obj.put("avg_price", metadata.avg_price); obj.put("c_date", new Date()); println obj.toString() thirdPartyColl.save(obj); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 3993f6663..1aae0a14c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -17,8 +17,8 @@ class SynchDistrictPrice { def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] - def sourceMap = ["58.com":"58同城","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多", - "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] + //"fang.com":"搜房", + def sourceMap = ["fang.com":"搜房"] static main(args) { def task = new SynchDistrictPrice() task.run() From 2a37912e7e4e6827b474e7431a179852b32aa388 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 17 Jan 2022 13:21:02 +0800 Subject: [PATCH 163/212] update --- .../com/fangjia/script/gj/level/DistrictIndexBatch.groovy | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index f2fbed25d..e0460cf88 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -64,6 +64,7 @@ class DistrictIndexBatch extends BaseBatch{ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = CommonDatautils.findAllCity(); citys.remove("上海") + citys.remove("北京") for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) // dists = dists[0..20] @@ -152,9 +153,9 @@ class DistrictIndexBatch extends BaseBatch{ def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] - def fileName = "/home/gj/potentiality.dat" + def fileName = "/home/gj/potentiality_${city}.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\评级\\上海综合潜力.dat" + fileName = "D:\\数据\\评级\\${city}综合潜力.dat" } FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", true); From 9100314a96e2591ce454eaf7070c8d3e9c3c2a48 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jan 2022 14:30:39 +0800 Subject: [PATCH 164/212] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=88=BF=E6=BA=90?= =?UTF-8?q?=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../ccbevaluation/ExportOracleData.groovy | 11 +- .../fangjia/script/ccbevaluation/Test.groovy | 88 +++- .../script/ccbevaluation/TradeStatLog.groovy | 16 +- .../ccbevaluation/data/CcbDistrictToDB.groovy | 89 +++- .../ccbevaluation/data/ExcelDataTest.groovy | 119 ++++- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../script/gj/level/DistrictIndexBatch.groovy | 9 +- .../script/gj/paimai/ChromeDriverPool.groovy | 61 +++ .../gj/paimai/LianjiaCrawlByChrome.groovy | 428 ++++++++++++++++++ .../fangjia/script/gj/paimai/Worker.groovy | 31 ++ .../script/thirdPriceCrawl/Esf58Crawl.groovy | 54 +-- .../trade/LianJiaTradeCrawl.groovy | 185 ++++++++ .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 14 files changed, 1007 insertions(+), 94 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 41273e6c3..eb62806e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211118, 20211219); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 736221526..bc608f48f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -11,7 +11,7 @@ import java.sql.ResultSet class ExportOracleData { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -24,11 +24,11 @@ class ExportOracleData { } def run() { - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // exportGuidedPrice() - exportCommunityGuidedPrice(lastMonth.toString()+"28") +// exportCommunityGuidedPrice(lastMonth.toString()+"28") // // //exportThirdPartyPrice()//废弃掉,老表 // exportThirdPartyPrice2(lastMonth) @@ -47,12 +47,13 @@ class ExportOracleData { // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) -// exportCommunityBasePrice(curMonth) + // // exportCityPrice(curMonth) // exportRegionPrice(curMonth) -// exportCommunityBasePriceGj(curMonth) +// exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0707c749c..15f1d38c1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -27,8 +27,8 @@ import java.util.concurrent.atomic.AtomicLong import java.util.regex.*; -def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") +//def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", +// "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") def sqlInstance236 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -271,7 +271,7 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def //},0) //def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充数据删除3658_1228.xlsx",{d-> +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d-> // if(d['A']){ // def udata = new BasicDBObject() // udata.put("name_cat", 11) @@ -284,26 +284,26 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // println index // } //},0) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> - if(d['A']){ - def region = d['B'] - def newRegion = RegionData.formatRegion(d['A'], region) - if(newRegion) { - def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") - if(dist){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", dist._id) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - }else{ - println d - } - } - } -},0) +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> +// if(d['A']){ +// def region = d['B'] +// def newRegion = RegionData.formatRegion(d['A'], region) +// if(newRegion) { +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +// if(dist){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", dist._id) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// }else{ +// println d +// } +// } +// } +//},0) //def citys = CcbCommonUtils.citys //def rets = [] @@ -488,6 +488,48 @@ ExcelUtils.modify("D:\\ // } //},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\小区信息修正的数据(3897)_0118.xlsx", { d -> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// if(d['K']){ +// def formatAddress = AddressUtil.formatAddress(d['B'], d['K']) +// def address = null +// if(formatAddress && formatAddress.endsWith("号")) { +// address = formatAddress +// } +// +// +// def data = new BasicDBObject(); +// data.put("m_date", new Date()) +// if(d['M']){ +// long glat = d['M'].toString().toDouble() * 10000000000L +// long glng = d['L'].toString().toDouble() * 10000000000L +// data.put("lat",glat) +// data.put("lng",glng) +// +// def baidu = BaiDuApiUtils.geoconv(d['L'].toString().toDouble(), d['M'].toString().toDouble(), 3) +// if(baidu){ +// long blng = baidu.x * 10000000000L +// long blat = baidu.y * 10000000000L +// data.put("lat2",blat) +// data.put("lng2",blng) +// } +// +// } +// +// if(address){ +// data.put("address", address) +// } +// index++ +// println index +" "+d['B'] + " "+data +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', data)) +// } +// } +// } +//}) + def findDealCount(sqlInstance, def startDate, def endDate){ def sql = "select cityname,count(*) as count from bigdata_deal_gj where dealdate>=to_date('${startDate}','yyyy-mm-dd') and dealdate + def city = d['A'] + def region = d['B'] + def newRegion = RegionData.formatRegion(city, region) + if(newRegion){ + def name = d['C'] + def alias = [] + if(d['F']){ + if(name != d['F']){ + alias << d['F'] + } + } + + def formatAddress = null + if(d['G']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['G']) + } + def address = null + if(d['D']){ + address = d['D'] + } + if(d['E']){ + address = address + d['E'] + } + + if(!formatAddress && address){ + formatAddress = AddressUtil.formatAddress(d['A'], address) + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['H']){ + glat = d['I'].toString().toDouble() * 10000000000L + glng = d['H'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['H'].toString().toDouble(), d['I'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + } + + if(name){ + def dist = CommonDatautils.findDistrict(d['A'], newRegion, name, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", d['A']) + data.put("region",newRegion) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + if(alias){ + data.put("alias",alias) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + println index+" "+data + } + } + } +// } + },0); } def run(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index 8ec8ba66a..e7411bf1b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -15,7 +15,7 @@ import com.mongodb.DBCollection DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d -> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K']) { // def rMap = SeaweedDataMatchProcess.doMatch(d['A'], newRegion, d['C'], d['D']+d['E']); @@ -32,7 +32,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price" //}); -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d-> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K']){ // @@ -63,7 +63,7 @@ DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price" // //},0) -//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\高德反查数据3_2003.xlsx",{d-> +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工审核_3016结果.xlsx",{d-> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K'] && d['G']){ // @@ -303,27 +303,100 @@ def city = " //}); -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> - if(d['K'] && d['K']=="别名(高德)" && d['G']){ +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\新增数据_1269.xlsx", { d -> +// if(d['K'] && d['K']=="别名(高德)" && d['G']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") +// def dist = null +// if(rets){ +// dist = rets[0] +// } +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['C'])){ +// alias << d['C'] +// } +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// } +// } +//}); + + + +//def rets = [] +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d -> +// if(d['A']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['E'])){ +// alias << d['E'] +// } +// if(alias && dist.mapping_alias!=alias){ +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// rets << [dist._id.toString(), d['E']] +// } +// } +// } +//}) +//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据") + + +def rets = [] +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d -> +// if(d['A']){ +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if(newRegion){ +// def dists = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district") +// def dist = null +// if(dists && dists.size()==1){ +// dist = dists[0] +// } +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(!alias.contains(d['D'])){ +// alias << d['D'] +// } +// if(alias && dist.mapping_alias!=alias){ +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// rets << [dist._id.toString(), d['D']] +// } +// } +// } +//}) +//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据") + + +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d -> + if(d['A']) { def newRegion = RegionData.formatRegion(d['A'], d['B']) - if(newRegion){ - def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['G'],"district") - def dist = null - if(rets){ - dist = rets[0] - } - if(dist){ - def alias = dist.mapping_alias - if(!alias){ - alias = [] - } - if(!alias.contains(d['C'])){ - alias << d['C'] - } - def udata = new BasicDBObject() - udata.put("mapping_alias",alias) - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + if (newRegion) { + def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district") + if (dist) { + rets << [dist._id.toString(), d['C']] } } } -}); +}) +ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index a1350cc2f..c43936a8a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,7 +48,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-11-01" + static def statDate = "2021-12-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2021年12月\\二手房\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index e0460cf88..05ed58021 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -244,9 +244,12 @@ class DistrictIndexBatch extends BaseBatch{ } } } - def avg_charge = rets.sum()/rets.size(); - avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge) - return avg_charge; + if(rets){ + def avg_charge = rets.sum()/rets.size(); + avg_charge = DataHandleUtils.getDoubleRound(2, avg_charge) + return avg_charge; + } + return null } def getPriceInRegionRank(d, nbLists, priceMap){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy new file mode 100644 index 000000000..53c9378bd --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/ChromeDriverPool.groovy @@ -0,0 +1,61 @@ +package com.fangjia.script.gj.paimai + +import org.openqa.selenium.chrome.ChromeDriver + +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicInteger + +public class ChromeDriverPool { + + private static ConcurrentHashMap pool ;//线程池 + private static int coreSize = 2;//核心线程池,一直驻留的资源 + private static int maxSize = 10;//TODO 设置最大数量资源,用于解决瞬时并发,瞬时并发过后要定时去识别空闲的释放掉,保留coreSize的资源 + private static AtomicInteger count = new AtomicInteger(0);//原子自增!! + + static { + //驱动初始化 + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + pool = new ConcurrentHashMap<>(); + } + + private ChromeDriverPool(){ + } + + public static void init(int coreSize){//线程池数量初始化 + ChromeDriverPool.coreSize = coreSize; + } + + public static Worker getPool(){ + if(count.getAndIncrement() < coreSize){//自增!!! + return push(new Worker()); + }else{ + return pop(); + } + } + + + public static Worker push(Worker worker){ + pool.put(count.get(),worker); + return worker; + } + + /** + * 循环获取未使用的资源 + * @return + */ + public static Worker pop(){ + Set> sets = pool.entrySet(); + Iterator> its = sets.iterator(); + while(its.hasNext()){ + Map.Entry workEntry = its.next(); + Worker work = (Worker)workEntry.getValue(); + if(work.status == 1){ + return work; + } + } + return null; + } + + +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy new file mode 100644 index 000000000..cba9e27b7 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -0,0 +1,428 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.gj.paimai.ChromeDriverPool +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +public class LianjiaCrawlByChrome { + + def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new LianjiaCrawlByChrome(); + task.run() + // task.run1() + } + + def run1(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def cityUrls = getAllCityUrls(driver) + getLogin(driver) + + for(def cityUrl : cityUrls){ + def detailUrls = getPageUrls(driver, cityUrl) + println cityUrl +" 详情链接:"+detailUrls.size() + poolCrawl( detailUrls) + } + } + + def poolCrawl(detailUrls){ + //模拟并发 + int parallelSize = 10; + final CountDownLatch latch = new CountDownLatch(1); + ExecutorService pool = Executors.newFixedThreadPool(parallelSize); + + for (int i = 0; i < detailUrls.size(); i++) { + final def detailUrl = detailUrls[i] + pool.execute(new Runnable() { + @Override + public void run() { + + Worker work = null; + try { + latch.await();//阻塞所有线程 + System.out.println(Thread.currentThread().getName()); + while (true) {//业务需求,一直等待直到获取到驱动 + work = ChromeDriverPool.getPool(); + if (work != null) { + break; + } + Thread.sleep(1000); + } + work.start(); + ChromeDriver driver = work.getDriver(); + def detail = crawling(driver, DEAL_PRICE_COLL, detailUrl) + println detail.city + " " + detail.region + " " + detail.district_name + System.out.println(Thread.currentThread().getName()+":"+driver); + Thread.sleep(1000 * 10); + } catch (Exception e) { + System.out.println(e.getMessage()); + }finally { + work.shutdown(); + } + } + }); + } + latch.countDown(); + } + + def run(){ +// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); +// System.setProperty("http.nonProxyHosts", "localhost"); +// WebDriver driver = new ChromeDriver() +// def cityMap = getAllCityUrls(driver) +// getLogin(driver) +// crawlData(driver, cityMap) + + addDealRegion(); + } + + def addDealRegion(){ + def q = new BasicDBObject() + q.put("region", null) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = DEAL_PRICE_COLL.find(q) + def keys = [] + for(def d : dbcursor){ + keys << ["city":d.city, "name":d.district_name] + } + for(def m : keys){ + //def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district") + def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district") + if(list && list.size()==1){ + def dist = list[0] + + def query = new BasicDBObject() + query.put("city", m.city) + query.put("district_name", m.name) + query.put("source", "链家在线") + query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) + + def data = new BasicDBObject() + data.put("region", dist.region) + def count = DEAL_PRICE_COLL.count(query) + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + } + + def crawlData(driver, cityMap){ + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + cityMap.each{city, cityUrl-> + //def detailUrls = getPageUrls(driver, cityUrl) + def detailUrls = getPageDeail(driver, city, cityUrl) + println cityUrl +" 详情链接:"+detailUrls.size() + // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + } + } + + def getPageUrls(driver, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + + def getPageDeail(driver, city, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + parse(city, e) + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + def parse(city,e){ + try { + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + detail.city = city + detail.district_name = arrs[0] + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + } + } + + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = e.select(".title a").attr("href") + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } + } + }catch(Exception ex){ + ex.printStackTrace() + println e.select(".title a").attr("href") + } + } + + + def findRegionByThirdPary(city, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("name", district_name) + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1) + def dbo = null; + for(def d : dbcursor){ + dbo = d + } + if(dbo){ + return dbo.region + } + return null; + } + def getContent(driver, detailUrl){ + driver.get(detailUrl) + def str = driver.getPageSource() + if(str && str.contains("登录")){ + getLogin(driver) + return getContent(driver, detailUrl) + } + return str; + } + def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){ + if(detailUrls){ + for(def i=0;i=4){ + detail.city = eles[0].text().replace("房产网","") + detail.region = eles[2].text().replace("二手房成交","") + detail.district_name = eles[eles.size()-1].text().replace("二手房成交","") + } + if(detail.district_name) { + def price = JQueryUtils.selectText(".dealTotalPrice i", doc) + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(DEAL_PRICE_COLL, detail) + if (!dbo) { + def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") + if (roomStr) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")) + } + detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") + detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") + def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "") + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = detailUrl + detail.c_date = new Date() + detail.m_date = new Date() + + DEAL_PRICE_COLL.save(detail) + } + } + return detail; + } + + + def findTrade(def detail){ + def q = new BasicDBObject() + q.put("city", detail.city) + q.put("region", detail.region) + q.put("district_name", detail.district_name) + q.put("area", detail.area) + q.put("total_price", detail.total_price) + q.put("trade_date", detail.trade_date) + q.put("source", detail.source) + def dbo = DEAL_PRICE_COLL.findOne(q) + return dbo + } + + def getAllCityUrls(driver){ + driver.get("https://www.lianjia.com/city/") + Thread.sleep(8000) + def cityDoc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select(".city_province a", cityDoc) + def cityMap = [:] + if(cityEles){ + for(def e : cityEles){ + def href = e.attr("href") + if(!href.contains("fang.lianjia")){ + cityMap.put(e.text(), href) + } + } + } + return cityMap + } + + def getLogin(driver){ + driver.get("https://clogin.lianjia.com/login?service=https%3A%2F%2Fwww.lianjia.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.lianjia.com%252Fchengjiao%252F"); + WebDriverWait wait = new WebDriverWait(driver,20) + Thread.sleep(10000) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + + + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + + + elemUsername.sendKeys("15021384995"); + + elemPassword.sendKeys("12qwaszx"); + + btn.submit(); // 点击行为; + } +} + + + + + + + + + + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy new file mode 100644 index 000000000..72d9c4559 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/Worker.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.gj.paimai + +import org.openqa.selenium.chrome.ChromeDriver + +class Worker { + private ChromeDriver driver; + /** + * 状态修改要让其他线程可见 + * 1就绪 2运行 + */ + private volatile int status; + + public Worker(){ + driver = new ChromeDriver(); + status = 1; + } + + + public void shutdown(){ + this.status = 1; + } + + + public void start(){ + this.status = 2; + } + + public ChromeDriver getDriver() { + return driver; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 10c39958e..46b9b6d4c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -42,35 +42,38 @@ class Esf58Crawl extends BaseBatch{ //"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", // "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", // "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", -// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","三亚":"sanya|2422","阜阳":"fy|2325", +// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", // "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", -// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385", - def cityList = ["北京":"bj|1","上海":"sh|2","深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","烟台":"yt|228","信阳":"xy|8694","南充":"nanchong|2378","洛阳":"luoyang|556","南阳":"ny|592","厦门":"xm|606", - "宜昌":"yc|858","张家口":"zjk|3328","昆明":"km|541","柳州":"liuzhou|7133","绵阳":"mianyang|1057", "泉州":"qz|291","济宁":"jining|450","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", - "淮北":"huaibei|9357","滁州":"chuzhou|10266", - "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", - "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", - "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", - "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", +// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", +// "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", +// "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", +// "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", +// "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", +// "淮北":"huaibei|9357","滁州":"chuzhou|10266", +// "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", +// "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", +// "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", +// "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752", + def cityList = ["武夷山":"wuyishan|10761", "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", - "漳浦":"zhangpu|717", "珠海":"zh|910","惠州":"huizhou|722","江门":"jm|629","汕头":"st|783","湛江":"zhanjiang|791", - "肇庆":"zq|901","茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389","清远":"qingyuan|7303", + "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", + "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", - "南宁":"nn|845","桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", - "遵义":"zunyi|7620","黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", - "海口":"haikou|2053","五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", - "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","保亭":"baoting|10367","白沙":"baish|10380", + "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", @@ -82,7 +85,7 @@ class Esf58Crawl extends BaseBatch{ "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", - "哈尔滨":"hrb|202","大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", + "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", @@ -93,21 +96,21 @@ class Esf58Crawl extends BaseBatch{ "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", - "长沙":"cs|414","株洲":"zhuzhou|1086","益阳":"yiyang|10198","常德":"changde|872","衡阳":"hy|914", - "湘潭":"xiangtan|2047","岳阳":"yy|821","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", + "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", + "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", - "保定":"bd|424","廊坊":"lf|772","秦皇岛":"qhd|1078","沧州":"cangzhou|652","邢台":"xt|751","衡水":"hs|993", + "邢台":"xt|751","衡水":"hs|993", "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", - "沙河":"shaheshi|755", "徐州":"xz|471", "扬州":"yz|637","淮安":"ha|968", - "泰州":"taizhou|693","宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", + "沙河":"shaheshi|755", + "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", @@ -123,7 +126,7 @@ class Esf58Crawl extends BaseBatch{ "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", "梨树县":"lishu|10176", - "鞍山":"as|523","锦州":"jinzhou|2354","抚顺":"fushun|5722", + "鞍山":"as|523","抚顺":"fushun|5722", "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", @@ -131,10 +134,10 @@ class Esf58Crawl extends BaseBatch{ "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", - "西宁":"xn|2052","海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", + "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", "泰安":"ta|686","聊城":"lc|882", - "枣庄":"zaozhuang|961","德州":"dz|728","日照":"rizhao|3177","菏泽":"heze|5632", + "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", @@ -170,8 +173,7 @@ class Esf58Crawl extends BaseBatch{ "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", - "宁波":"nb|135","金华":"jh|531", - "绍兴":"sx|355","湖州":"huzhou|831","丽水":"lishui|7921","衢州":"quzhou|6793", + "丽水":"lishui|7921","衢州":"quzhou|6793", "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy new file mode 100644 index 000000000..6540e9ca0 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/trade/LianJiaTradeCrawl.groovy @@ -0,0 +1,185 @@ +package com.fangjia.script.thirdPriceCrawl.trade + +import com.fangjia.httpclient.UserAgents +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.util.ABuYunUtil +import com.fangjia.script.util.HttpClient +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils +import org.apache.http.Header +import org.apache.http.HttpResponse +import org.apache.http.client.methods.HttpGet +import org.apache.http.impl.client.HttpClients +import org.apache.http.client.HttpClient + +class LianJiaTradeCrawl { + + def static final LOGIN_URL = "https://clogin.lianjia.com/authentication/authenticate" + + public def CHARSET = "utf-8"; + public def ERROR_MSG = "访问过于频繁"; + + static main(args) { + def task = new LianJiaTradeCrawl() + task.run() + } + + def run(){ + def valueMap = getUidAndSid(); + def xqMap = goToXiaoqu(valueMap); + // goLogin() + } + + def getUidAndSid(){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://sh.lianjia.com/"); + get.setHeader("Accept-Encoding", "gzip, deflate, br") + get.setHeader("Accept-Language", "zh-CN,zh;q=0.9") + get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + HttpResponse response = client.execute(get); + def valueMap = getUidAndSid(response) + return valueMap; + + } + + def goToXiaoqu(valueMap){ + HttpClient client = HttpClients.createDefault(); + HttpGet get = new HttpGet("https://sh.lianjia.com/xiaoqu/"); + get.setHeader("Host","sh.lianjia.com") + get.setHeader("Connection","keep-alive") + get.setHeader("sec-ch-ua-mobile","?0") + get.setHeader("sec-ch-ua-platform","Windows") + get.setHeader("Upgrade-Insecure-Requests","1") + get.setHeader("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + get.setHeader("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9") + get.setHeader("Sec-Fetch-Site","same-origin") + get.setHeader("Sec-Fetch-Mode","navigate") + get.setHeader("Sec-Fetch-User","?1") + get.setHeader("Sec-Fetch-Dest","document") + get.setHeader("Referer","https://sh.lianjia.com/") + get.setHeader("Accept-Encoding","gzip, deflate, br") + get.setHeader("Accept-Language","zh-CN,zh;q=0.9") + + def cookieStr = "" + valueMap.each {k,v-> + cookieStr = cookieStr +k +"="+v+";" + } + get.setHeader("Cookie", cookieStr) + + HttpResponse response = client.execute(get); + def newVvalueMap = getUidAndSid(response) + return valueMap; + } + + + def getUidAndSid(HttpResponse response) { + Header[] headers = response.getHeaders("Set-Cookie"); + def valueMap = [:] + for(def header : headers){ + String[] split = header.getValue().split(";"); + for (String s : split) { + def arrs = s.split("=") + valueMap.put(arrs[0], arrs[1]) + } + } + return valueMap; + } + + + + def goLogin() { + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","clogin.lianjia.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","1415") + headers.put("sec-ch-ua",'Not A;Brand";v="99", "Chromium";v="96", "Google Chrome";v="96"') + headers.put("sec-ch-ua-mobile","?0") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36") + headers.put("sec-ch-ua-platform","Windows") + headers.put("Content-Type","application/json") + headers.put("Accept","*/*") + headers.put("Sec-Fetch-Site","same-origin") + headers.put("Sec-Fetch-Mode","cors") + headers.put("Sec-Fetch-Dest","empty") + headers.put("Cookie","Cookie: lianjia_uuid=45384ffe-4245-46fb-aa3f-aa6931a0a392; lianjia_ssid=75aa3a46-9d5f-42a9-b39b-4fc5e181e37c; sensorsdata2015jssdkcross=%7B%22distinct_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22%24device_id%22%3A%2217e486c27c77-0d010c318a1f7-4303066-2073600-17e486c27c9242%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_referrer_host%22%3A%22%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%7D%7D; select_city=350100; crosSdkDT2019DeviceId=-y4aedm--2s8wtc-85pqx15zlsvkewy-gkpcb9u2t") + + headers.put("Accept-Encoding", "gzip, deflate, br") + headers.put("Accept-Language", "zh-CN,zh;q=0.9") +// headers.put("User-Agent", UserAgents.get()) + headers.put("Referer", "https://clogin.lianjia.com/authentication/sdk/init?version=2.0") + headers.put("Origin", "https://clogin.lianjia.com") + //params信息 + Map params = new LinkedHashMap() + params.put("mainAuthMethodName", "username-password") + params.put("srcId", "eyJ0Ijoie1wiZGF0YVwiOlwiMWM1ZDg5ZTFjOTc0ZWViODliOTJkYTE1ZWEzOWM5ZDY1MWNlODJkMjkzYzQzYmY2NjRiMWEyYWI2NDRkYmViY2RiMDVkNmY3YTc3MmVkODBlZjNhMjBjYWM5ODVlNjY0ODc3YmVjMTBjYTRkM2RkNWFhOTY0Mjc0OTliNzVkZWU4ZDU1YzVkNzA1ZTRhMzcyY2U1MGZkMGVmZDE0OWY0NzYxMzk4MjRlMjU0Yzk0MzY0MDJjMjdkYjkxODdjNDliOWU4MWJlNWJjNGY2ZmEyNmY5Yjk2NzAxOTFkZDQwOWMxMzhjNTEwOTUwMmQ1OGE4NTE0Y2IzMzgxNzQ0ZTgwMzg2NmVjOGZhOWIyNjc1MzEyMTQ5YjE2NjU3ZjEyNDYyY2RiOTc4ZjM2ODVlNWVmYWViMGU1YjEwZmFiMWUzNDFhMzg4NmYzZmMwM2FhZWNjZDg1ZmIwNGYzZDQ2NmRjNlwiLFwia2V5X2lkXCI6XCIxXCIsXCJzaWduXCI6XCIzYjYwYTYwOFwifSIsInIiOiJodHRwczovL2Nsb2dpbi5rZS5jb20vbG9naW4/c2VydmljZT1odHRwcyUzQSUyRiUyRnd3dy5rZS5jb20lMkZ1c2VyJTJGY2hlY2tsb2dpbiUzRnJlZGlyZWN0JTNEaHR0cHMlMjUzQSUyNTJGJTI1MkZmei5rZS5jb20lMjUyRmNoZW5namlhbyUyNTJGYzM0MjAwMzI1MDU1MzAxMDElMjUyRiIsIm9zIjoid2ViIiwidiI6IjAuMSJ9") + params.put("version", "2.0") + params.put("accountSystem", "customer") + params.put("service", "https://www.lianjia.com/user/checklogin?redirect=https%3A%2F%2Ffz.lianjia.com%2Fchengjiao%2F") + params.put("loginTicketId", "nM7eXE8ZWBP67aiapHqZRpn0QHTfCX4q") + + + Map credentialparams = new LinkedHashMap() + credentialparams.put("username", "15021384995") + credentialparams.put("password", "dx/vjEHyEc8ETOqKDn5ZhNQZuO5JjoCu1D0UXVTQ2UrOE/liQYeNcj9H9bPB5c43UDRyKZSvA9UzBc4ycTOkDkq7pgITkJrzP0Iagkj19R0TT/yX/9NHEwm/GaqBfVvfJXfmw6ZX9vCYLaOaiOJ31PTpI5hDUS/PxxHBkmqC7Bg=") + credentialparams.put("ioaRunState", false) + credentialparams.put("ioaMid", "{}") + credentialparams.put("alertSt", "") + credentialparams.put("ioaStateVersion", 2) + credentialparams.put("encodeVersion", "1") + params.put("credential", credentialparams) + + Map contextparams = new LinkedHashMap() + contextparams.put("deviceId", "default") + contextparams.put("sign", "default") + params.put("context", contextparams) + + //post请求 + responseMap = isProxyAndTryTimesToRequest(LOGIN_URL, HttpClient.CHARSET, 100, "POST", params, headers, true) + } catch (Exception e) { +// println "goLogin(${username}, ${password}, ${rememberme})发生异常, 异常信息为:" + e.getMessage() + } + return responseMap + } + + + def static isProxyAndTryTimesToRequest(url, charset, tryTimes, requestType, params, headers, isJson) { + def responseMap = [:] + def _tryTimes_ = 0 + //请求响应为空(切换代理重试) + while (_tryTimes_ < tryTimes) { + try { + //第一次请求不提示重试 +// println("请求地址URL为:" + url + "代理尝试次数tryTimes为" + (_tryTimes_ + 1)) + if (StringUtils.isNotEmpty(requestType) && requestType.equals("POST")) { + responseMap = ABuYunUtil.doPostResponse(url, params, headers, charset, isJson) + } else { + responseMap = ABuYunUtil.doGetResponse(url, params, headers, charset) + } + def response = responseMap.get("response") + if (StringUtils.isEmpty(response)) { +// println('isProxyAndTryTimesToRequest 请求response为空') + //控制sleep时间 + //Thread.sleep(sleepTimes); + } else { + //控制sleep时间 + //Thread.sleep(sleepTimes); +// println('isProxyAndTryTimesToRequest 请求response响应成功, 响应内容为:' + response) + break + } + } catch (Exception e) { +// println("isProxyAndTryTimesToRequest请求发生异常, 请求url为: " + url + "请求异常信息为:" + e.getMessage()) + } finally { + //统计所有请求总次数(后期统计用) +// println("目前整个" + GLOBAL_WEBSITE_SOURCE + "抓取请求总次数count为: " + count.incrementAndGet()) + _tryTimes_++ + } + } + responseMap.put("tryTimes", _tryTimes_) + responseMap.put("url", url) + return responseMap + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index d1d4ada4c..59ad4ae49 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20211101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20211201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20211201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220101", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 From c1379169d230a473b89782c27c917caa43789132 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jan 2022 14:35:52 +0800 Subject: [PATCH 165/212] =?UTF-8?q?100-=E3=80=8B236?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index eb62806e5..6448e0d67 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -57,7 +57,7 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:31521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { From 31e1dea24c71240d65fd69d019a5eb209af00cef Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jan 2022 14:41:51 +0800 Subject: [PATCH 166/212] =?UTF-8?q?31521-=E3=80=8B1521?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 6448e0d67..7138e276c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -57,7 +57,7 @@ class BigdataForsaleBatch extends BaseBatch{ public static final String[] TAGS = ["独栋", "联排", "叠加", "双拼", "花园面积", "泳池", "别墅", "带花园", "洋房", "叠拼", "独院", "联体", "微墅", "大平墅"]; public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(60) - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { From 431ae4aa1b1cfd609ac4f20a2e215476ce2ff985 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jan 2022 16:38:42 +0800 Subject: [PATCH 167/212] =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=E6=97=B6=E9=97=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/ExportOracleData.groovy | 34 +++++++++---------- .../ccbevaluation/StagingDistrictBatch.groovy | 2 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 ++-- .../gj/paimai/LianjiaCrawlByChrome.groovy | 14 +++++--- 5 files changed, 31 insertions(+), 27 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 39b653a2f..e38ff62f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -42,7 +42,7 @@ class BigdataDealBatch extends BaseBatch{ DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index bc608f48f..b3f07018c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -24,36 +24,36 @@ class ExportOracleData { } def run() { - def curMonth = 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); -// exportGuidedPrice() -// exportCommunityGuidedPrice(lastMonth.toString()+"28") -// + + //不会变 + exportEvaluationRateData() + exportBigdataPoiData() + exportBigdataDistrictData() + + exportGuidedPrice()//随人工改变 + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 + + //处理小区情况 + exportBigdataCommunity() + exportCommunityStaging() + exportCommunityBuileType() + // //exportThirdPartyPrice()//废弃掉,老表 // exportThirdPartyPrice2(lastMonth) -// -// -// // exportBigdataForsale("2021-11-19") -// -// exportEvaluationRateData() -// exportBigdataPoiData() -// exportCommunityStaging() -// exportCommunityBuileType() -// exportBigdataDistrictData() -// exportBigdataCommunity() -// exportBigdataDeal("2021-11-01") +// exportBigdataDeal("2021-11-01") // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) -// // exportCityPrice(curMonth) // exportRegionPrice(curMonth) // exportCommunityBasePriceGj(curMonth)//已合下面的合并 - exportCommunityBasePrice(curMonth) +// exportCommunityBasePrice(curMonth) // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy index 6f9584e40..532a96c43 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/StagingDistrictBatch.groovy @@ -40,7 +40,7 @@ class StagingDistrictBatch extends BaseBatch{ results << headers results.addAll(rows) - ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") + //ExcelUtils.write("d:\\数据\\建行\\房产估值\\数据文件\\excel\\COMMUNITY_STAGING_GJ.xlsx", results, "数据") FileUtils.batchCreateFile(rows, "d:\\数据\\建行\\房产估值\\数据文件\\txt\\COMMUNITY_STAGING_GJ.dat",headers, "utf-8", false); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 32bc302ae..ca4add802 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202111//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202109 - def endDate = 202111 + def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202110 + def endDate = 202112 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index cba9e27b7..fe25a7538 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -35,6 +35,7 @@ public class LianjiaCrawlByChrome { def task = new LianjiaCrawlByChrome(); task.run() // task.run1() + task.addDealRegion(); } def run1(){ @@ -100,14 +101,16 @@ public class LianjiaCrawlByChrome { // getLogin(driver) // crawlData(driver, cityMap) - addDealRegion(); + } def addDealRegion(){ + def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + lastMonth = lastMonth +"-01" def q = new BasicDBObject() q.put("region", null) q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) - q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) def dbcursor = DEAL_PRICE_COLL.find(q) def keys = [] for(def d : dbcursor){ @@ -121,13 +124,14 @@ public class LianjiaCrawlByChrome { def query = new BasicDBObject() query.put("city", m.city) + query.put("region", null) query.put("district_name", m.name) - query.put("source", "链家在线") - query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) +// query.put("source", "链家在线") + query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd"))) def data = new BasicDBObject() data.put("region", dist.region) - def count = DEAL_PRICE_COLL.count(query) + def ds = DEAL_PRICE_COLL.count(query) DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) } } From dfb6a6efac68994928fbb35e5ed8d5f56a98147f Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jan 2022 16:43:23 +0800 Subject: [PATCH 168/212] =?UTF-8?q?100-=E3=80=8B236?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index ca4add802..10f845a3e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -33,7 +33,7 @@ class ThirdPartyPriceBatch extends BaseBatch { def sourceMap = ["诸葛找房":"01", "安居客":"02","房天下":"03","链家":"04","城市家居":"05","麦田":"06","贝壳":"07","淘房365":"08","中原":"09","兔博士":"10","Q房":"11","房探":"12", "房多多":"13","我爱我家":"14","禧泰数据":"15","吉屋网":"16","乐居":"17","21世纪":"18","房产超市":"19","58同城":"20"] - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { From 5c6e8f6f8d3640c0a25ee83207cf94f5219ba019 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 19 Jan 2022 17:48:25 +0800 Subject: [PATCH 169/212] =?UTF-8?q?=E4=BC=B0=E5=80=BC=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/ExportOracleData.groovy | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index b3f07018c..883184881 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -11,11 +11,7 @@ import java.sql.ResultSet class ExportOracleData { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") - - - def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") static main(args) { @@ -41,22 +37,25 @@ class ExportOracleData { exportCommunityStaging() exportCommunityBuileType() -// //exportThirdPartyPrice()//废弃掉,老表 -// exportThirdPartyPrice2(lastMonth) -// exportBigdataForsale("2021-11-19") + exportThirdPartyPrice2(lastMonth) + exportBigdataForsale("2021-12-19") + exportBigdataDeal("2021-12-01") -// exportBigdataDeal("2021-11-01") + + //计算结果相关 // exportAvmEvaluationGJNB(curMonth) // exportAVMEvaluationItemGJNB(curMonth) - +// // exportCityPrice(curMonth) // exportRegionPrice(curMonth) - +// // exportCommunityBasePriceGj(curMonth)//已合下面的合并 // exportCommunityBasePrice(curMonth) - +// // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) +// +// //ok文件 // exportLastFile() } From f7f5ad1acf81f209424714c4fd8c3aef945b4f82 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 24 Jan 2022 11:11:41 +0800 Subject: [PATCH 170/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E6=88=BF=E9=87=91?= =?UTF-8?q?=E9=83=A8=E8=87=AA=E5=8A=A8=E6=A3=80=E6=B5=8B=E6=9C=BA=E5=99=A8?= =?UTF-8?q?=E4=BA=BA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/SnapshotMonitor.groovy | 2 ++ .../script/temp/shenjian/poi/GetPoiFromExcel.groovy | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy index fecb21729..e638088fe 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/SnapshotMonitor.groovy @@ -19,6 +19,8 @@ class SnapshotMonitor extends BaseBatch { // 运维组 运维警报 static final access_token = "2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" + // 房金部自动检测机器人 + // https://oapi.dingtalk.com/robot/send?access_token=1b6d0d782d3c316f2d3b6b18ea5144816170dd392befbf2042a24a01fb84b736 static void main(String[] args) { runBatch(new SnapshotMonitor(), args) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy index d2644582e..bd49875b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetPoiFromExcel.groovy @@ -19,18 +19,18 @@ class GetPoiFromExcel { static void main(String[] args) { AtomicLong atomicLong = new AtomicLong(0) - List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false); + //List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\POI"), null, false); + List fileList = FileUtils.listFiles(new File("D:\\Users\\Domi\\Desktop\\房价网\\项目\\建行\\模型级联\\高德反查无ID"), null, false); for (File file : fileList) { List> seaweeds = ExcelUtils.read(file.getPath()) - final cityTemp = file.getName().substring(0, file.getName().indexOf("高德")) for (List infoTemp : seaweeds) { final List info = infoTemp executorService.submit(new Runnable() { @Override void run() { - String city = cityTemp - String region = info.get(2) - String seaweedName = info.get(3) + String city = info.get(0) + String region = info.get(1) + String seaweedName = info.get(2) List pois = GaoDeUtils.searchPoi(city, seaweedName, region, "120300|120301|120302"); if (CollectionUtils.isNotEmpty(pois)) { String jsonStr = JSONObject.toJSONString(pois) From 1aa30203610c290fe99f347e1c68eb464d49ef7b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Feb 2022 09:53:37 +0800 Subject: [PATCH 171/212] update --- .../DealAndThirdPriceDataProcess.groovy | 64 +++++- .../ccbevaluation/ExportOracleData.groovy | 32 +-- .../fangjia/script/ccbevaluation/Test.groovy | 24 +-- .../script/ccbevaluation/TradeStatLog.groovy | 18 +- .../ccbevaluation/data/ExcelDataTest.groovy | 77 +++++-- .../data/MatchDealPriceRegion.groovy | 78 +++++++ .../ronghe/ExportDistrictRongHe.groovy | 191 +++++++++--------- .../script/commData/CommonDatautils.groovy | 2 - .../script/fuzhou/FzDataResultStat.groovy | 144 +++++++++++++ .../script/gj/level/DistrictIndexBatch.groovy | 11 +- 10 files changed, 485 insertions(+), 156 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy index e21a8556a..f75d28656 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DealAndThirdPriceDataProcess.groovy @@ -13,8 +13,7 @@ import org.bson.types.ObjectId */ class DealPriceDataProcess { - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", - "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + def sqlInstance2 = Sql.newInstance("jdbc:oracle:thin:@192.168.0.94:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -31,16 +30,25 @@ class DealPriceDataProcess { } def delDealByPe(){ - def yearMonth = 202112 - def thirdMonth = 202111 - def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202112_1223 - 副本.xlsx",0,1) + def yearMonth = 202201 + def thirdMonth = 202112 + +// def rets = findAllDealByPe(yearMonth, 70) +// for(def r : rets){ +// deleteDealPrice(r) +// } + def list = findAllThirdByPe(yearMonth, 70) + for(def r : list){ + deletThirdPrice(r, thirdMonth) + } + def datas = ExcelUtils.read("D:\\数据\\建行\\房产估值\\估值模型测试202201_1221 - 副本.xlsx",0,1) for(def d : datas){ - if(d['G']){ - def rets = findDealByPe(d['A'], yearMonth, d['G']) - for(def r : rets){ - deleteDealPrice(r) - } - } +// if(d['G']){ +// def rets = findDealByPe(d['A'], yearMonth, d['G']) +// for(def r : rets){ +// deleteDealPrice(r) +// } +// } // if(d['H']){ // def rets = findCityByPpeRange(d['A'], yearMonth, 15, 20) @@ -161,6 +169,26 @@ class DealPriceDataProcess { return count } + def findAllDealByPe(yearMonth, peVal){ + def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='成交' and pe>${peVal}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.cityname = r.cityname + map.cityid=r.cityid + map.distid = r.distid + map.commid = r.commid + map.commname = r.commname + map.deal_price = r.target_avg_price + map.unit_price = r.unit_price + map.buildarea = r.buildarea + map.ppe = r.pe + map.dealdate = r.dealdate + rets << map + }) + return rets + } + def findDealByPe(city, yearMonth, peVal){ def sql = "select cityid,cityname,distid,commid,commname,unit_price,target_avg_price,pe,buildarea,to_char(dealdate,'yyyy-mm-dd') as dealdate from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='成交' and pe>${peVal}"; def rets = [] @@ -181,6 +209,20 @@ class DealPriceDataProcess { return rets } + + def findAllThirdByPe(yearMonth, peVal){ + def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and data_type='第三方' and pe>${peVal}"; + def rets = [] + sqlInstance236.eachRow(sql,{r-> + def map = [:] + map.year_month = r.year_month + map.commid = r.commid + map.target_avg_price = r.target_avg_price + rets << map + }) + return rets + } + def findThirdByPe(city, yearMonth, peVal){ def sql = "select year_month, commid, target_avg_price from avm_evaluation_item_gj where year_month=${yearMonth} and cityname='${city}' and data_type='第三方' and pe>${peVal}"; def rets = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 883184881..cb4fdc1e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -24,21 +24,21 @@ class ExportOracleData { def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - //不会变 - exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() - - exportGuidedPrice()//随人工改变 - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 - - //处理小区情况 - exportBigdataCommunity() - exportCommunityStaging() - exportCommunityBuileType() +// //不会变 +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() +// +// exportGuidedPrice()//随人工改变 +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 +// +// //处理小区情况 +// exportBigdataCommunity() +// exportCommunityStaging() +// exportCommunityBuileType() exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2021-12-19") +// exportBigdataForsale("2021-12-19") exportBigdataDeal("2021-12-01") @@ -879,7 +879,7 @@ class ExportOracleData { // // for(def city : citys) { // def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj"; + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] @@ -927,7 +927,7 @@ class ExportOracleData { allRows << data }); println allRows.size() - FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); // } println "community_p_describe_gj 数据文件输出完成......" CcbFtpUtils.ftpUpload(fileName, curDate) @@ -948,7 +948,7 @@ class ExportOracleData { for(def city : citys){ def cityCode = cityCodeDataMap.get(city) - def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}'";//price_date=${month} and + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 15f1d38c1..53a43fb77 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -//def fileName = "D:\\数据\\建行\\房产估值\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2021-11-01", "2021-12-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202112) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['R'] = oldcount -// def casecount = testMap.get(d['C']) -// d['S'] = casecount -// println d -// } -//},0) +def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01") +def testMap = findCaluateCityPPE(sqlInstance236, 202201) +ExcelUtils.modify(fileName,{d-> + if(d['C']){ + def oldcount = dealMap.get(d['C']) + d['T'] = oldcount + def casecount = testMap.get(d['C']) + d['U'] = casecount + println d + } +},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 32827c5ca..1b9d19e4f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -39,8 +39,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 - // task.exportTrade()//导出未入库成交 +// task.run()//统计最近3个月抓取的成交数据量 + task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -129,7 +129,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\数据文件\\12月成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据") println "表格生成完成" } @@ -479,16 +479,18 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers + def startDate = "2021-12-01" + def endDate = "2022-01-01" for(def city : citys){ - def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern("2021-10-01","yyyy-MM-dd"), DateUtils.parseFromTightByPattern("2021-11-01","yyyy-MM-dd")) + def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) // def trades = getYouDaData() //def statMap = getDistrictStatByCity(city, weekend) def nameMap = getAllDistrict(city) - def rMap = findDealIds(city); + def rMap = findDealIds(city, startDate, endDate); def ids = rMap.ids def details = rMap.detail @@ -559,7 +561,7 @@ class TradeStatLog { } println city +" "+rets.size() } - ExcelUtils.write("d:\\数据\\成交未入库记录(10月).xlsx", rets,"数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据") } def deleteDistrict(){ @@ -615,8 +617,8 @@ class TradeStatLog { return trades } - def findDealIds(city){ - def sql = "select * from bigdata_deal_gj where dealdate>=to_date('2021-10-01','yyyy-mm-dd') and dealdate diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index e7411bf1b..8a8a8b6c8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -11,10 +11,14 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import groovy.sql.Sql DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") +def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + //ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d -> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K']) { @@ -332,7 +336,7 @@ def city = " //def rets = [] //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\国信达名字加入映射别名.xlsx", { d -> -// if(d['A']){ +// if(d['A'] && d['E'].toString().contains("鑫康花园北区")){ // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion){ // def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") @@ -357,7 +361,7 @@ def city = " //ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名.xlsx", rets,"数据") -def rets = [] +//def rets = [] //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\查询别名表,将国信达数据增加到映射别名表里.xlsx", { d -> // if(d['A']){ // def newRegion = RegionData.formatRegion(d['A'], d['B']) @@ -388,15 +392,64 @@ def rets = [] //ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\新增映射别名_2.xlsx", rets,"数据") -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d -> - if(d['A']) { - def newRegion = RegionData.formatRegion(d['A'], d['B']) - if (newRegion) { - def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district") - if (dist) { - rets << [dist._id.toString(), d['C']] - } +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\修改状态的数据.xlsx", { d -> +// if(d['A']) { +// def newRegion = RegionData.formatRegion(d['A'], d['B']) +// if (newRegion) { +// def dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district") +// if (dist) { +// rets << [dist._id.toString(), d['C']] +// } +// } +// } +//}) +//ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据") + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d-> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['B']) +// if(dist){ +// d['E'] = dist.city +// d['F'] = dist.region +// d['G'] = dist.name +// d['H'] = dist.estate_type2 +// d['I'] = dist.name_cat +// d['J'] = dist.status +// } +// } +//}) + + +//ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\修改后结果数据.xlsx",{d-> +// if(d['B']) { +// def sql = "select count(*) as count from bigdata_community_gj where commid='${d['B']}'"; +// def count = 0; +// sqlInstance.eachRow(sql, { r -> +// count = r.count +// }) +// if(count>=1){ +// d['J'] = "oracle存在" +// } +// } +//}) + +def index = 0 +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d -> + if (d['B']) { + def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"]) + if(dist){ + def alias = dist.mapping_alias + if(!alias){ + alias = [] + } + if(alias && !alias.contains(d['C'])){ + index++ + println index + alias << d['C'] + def udata = new BasicDBObject() + udata.put("mapping_alias",alias) + DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + } } } -}) -ExcelUtils.write("D:\\数据\\建行\\房产估值\\小区\\映射别名存在——修改状态.xlsx", rets,"数据") +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy new file mode 100644 index 000000000..c52ecb26b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy @@ -0,0 +1,78 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.bson.types.ObjectId + +class MatchDealPriceRegion { + + static DBCollection collection = MongoUtils.getCollectionByName("fangjia_craw", "third_party_price", "192.168.0.235", 27018) + + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new MatchDealPriceRegion(); + task.run() + } + + def run(){ + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\成交\\部分未匹配成交.xlsx",{d-> + + def newRegion = null + if(d['C']){ + newRegion = RegionData.formatRegion(d['B'], d['C']) + } + if(!newRegion){ + index++ + newRegion = findRegionByThirdPrice(d['B'], d['D']); + if(!newRegion || (newRegion && newRegion==d['C'])){ + def rets = CommonDatautils.findDistrictsByName(d['B'], d['D'],"district") + if(rets && rets.size()==1){ + newRegion = rets[0].region + } + } + if(newRegion && newRegion!=d['C']){ + def query = new BasicDBObject() + query.put("_id", new ObjectId(d['A'])) + + def data = new BasicDBObject() + data.put("region", newRegion) + data.put("m_date", new Date()) + println index+" "+d['A']+" "+d['C']+" "+newRegion + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + }) + } + + def findRegionByThirdPrice(def city, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("name", name) + def dbcursor = collection.find(q).sort(new BasicDBObject("s_date",-1)).limit(10) + def map = [:] + for(def d : dbcursor){ + def count = map.get(d.region) + if(!count){ + count = 0 + } + map.put(d.region, count+1) + } + if(map){ + if(map.size()>1){ + map = map.sort(){ a, b -> + b.value.compareTo(a.value) + } + } + def key = map.keySet().getAt(0) + return key + } + return null + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index 683312af1..a027b584e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -27,15 +27,18 @@ class ExportDistrictRongHe extends BaseBatch{ def citys = CcbCommonUtils.citys; def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() - //检查ok文件是否上传 + +// //检查ok文件是否上传 // exportAllBigdataDealFJW() for(def city : citys) { def cityCode = cityCodeDataMap.get(city) +// +// exportBigdataDealFJW(cityCode, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) +// exportBigdataCommunityGJRH(cityCode) exportBigdataBuildingFJW(cityCode) exportBigdataCommfeeGj(cityCode) - - exportBigdataDealFJW(cityCode, '2021-11-01') +// exportBigdataCommbuildstruGJ(cityCode) exportBigdataCommhousetypeGJ(cityCode) exportBigdataCommbuildyearGj(cityCode) @@ -49,6 +52,10 @@ class ExportDistrictRongHe extends BaseBatch{ } exportEvaluateRate() exportOk() +// +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh ${curDate}")); } @@ -319,105 +326,109 @@ class ExportDistrictRongHe extends BaseBatch{ def stop = false def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" def index = 0 + + def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd") + def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd") + while(!stop){ index++ - def dealStartStr = startDateStr + def curStartStr = curStartDateStr - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat" + def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${dealStartStr.replace("-","")}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } - - def dealEndStr = DateUtils.add(startDateStr, 1) - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${dealStartStr}','yyyy-MM-dd') and dealdate - def data = [] - data << r.CITYID - data << r.DEALID - data << r.CITYNAME - data << r.DISTID - data << r.DISTNAME - data << r.STREET - data << r.STREETNO - data << r.COMMID - data << r.COMMNAME - data << r.BUILDID - data << r.BUILDNO - data << r.BUILDNAME - data << r.UNITID - data << r.UNITNO - data << r.UNITNAME - data << r.HOUSEID - data << r.ADDRESS - data << r.BUILDYEAR - data << r.PROPTYPE - data << r.PROPYEARS - data << r.PERIODNUM - data << r.FLOORNUM - data << r.FLOORTYPE - data << r.FLOORCNT - if(r.BUILDTYPE){ - def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") - data << arrs[0].trim() - }else{ - data << '' - } - data << r.BUILDSTRU - data << r.BUILDAREA - data << r.LIVINGAREA - data << r.HOUSESTYLE - data << r.ROOMCNT - data << r.LIVINGROOMCNT - data << r.KITCHENCNT - data << r.BATHROOMCNT - data << r.HOUSEUSE - data << r.HOUSESTRU - data << r.HOUSEASPECT - data << r.FITMENT - data << r.EVALUATENO - data << r.DEALUNITPRICE - data << r.DEALTOTALPRICE - data << r.DECLAREPRICE - data << r.EVALUATEPRICE - data << r.CONTRACTNO - data << r.DEALDATE - data << r.DEALTYPE - data << r.PAYTYPE - data << r.DEALPERIOD - data << r.IFELEVATOR - data << r.COMPAREAGARDEN - data << r.COMPAREATERRACE - data << r.COMPAREABALCONY - data << r.COMPAREABAYWINDOW - data << r.COMPAREAATTIC - data << r.COMPAREAGARAGE - data << r.COMPAREABASEMENT - data << r.COMPAREAOTHER - data << r.DATASOURCE - data << r.AGENCYID - data << r.AGENCYNAME - data << r.REMARK - data << r.CREATETIME1 - data << r.UPDATETIME1 - data << r.SOURCECODE - data << r.WORKDATE - allRows << data - }); - - if(dealStartStr == endDateStr){ + if(index==1){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')"; + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + if(curStartDateStr == curSndDateStr){ stop = true } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") - println "BIGDATA_DEAL_FJW 上传Ftp完成......${dealStartStr}" - startDateStr = dealEndStr + println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" + def curEndStr = DateUtils.add(curStartStr, 1) + curStartDateStr = curEndStr } } def exportAllBigdataDealFJW(){ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ"; + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE def data = [] @@ -487,15 +498,15 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.AGENCYNAME data << r.REMARK data << r.CREATETIME1 - data << r.UPDATETIME2 + data << r.UPDATETIME1 data << r.SOURCECODE data << r.WORKDATE allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/All/FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\All\\FJW_000000_All_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); println "BIGDATA_DEAL_FJW 导出完成......" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 23945eac2..63732e34f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -829,8 +829,6 @@ class CommonDatautils { keys.put("lat", 1); keys.put("lng", 1); keys.put("name_cat", 1); - keys.put("name_cat", 1); - keys.put("name_cat", 1); DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy new file mode 100644 index 000000000..46ad15c1b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -0,0 +1,144 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.Bytes +import com.mongodb.DBCollection +import com.mongodb.DBCursor +import com.mongodb.DBObject +import org.apache.commons.io.FileUtils + +/** + * 福州数据结果统计 + */ +class FzDataResultStat { + + public static final DBCollection REGION_STAT_COLL = MongoUtils.getCollectionByName("fangjia", "region_stat", "mongo.db.fangjia.com") + + static main(args) { + def task = new FzDataResultStat() + //task.priceMarkupRun(); + task.regionPriceTrendRun() + } + def priceMarkupRun(){ + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + + List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) + def map1 = [:] + for(def s : list1){ + def row = s.split("@\\|@") + def key = row[2]+"##"+row[3]+"##"+row[4] + map1.put(key, row[5]) + } + + List list2 = FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${lastMonth}.csv")) + def map2 = [:] + for(def s : list2) { + def row = s.split("@\\|@") + def key = row[2] + "##" + row[3] + "##" + row[4] + map2.put(key, row[5]) + } + def rows = [] + map1.each {k,v-> + def row = [] + def keys = k.split("##") + def lastPrice = map2.get(k) + def markup = null + if(v && lastPrice){ + markup = (v - lastPrice)*100.0/lastPrice + } + rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup] + } + ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") + + + def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; + + } + + + def regionPriceTrendRun(){ + def city = "福州" + def add = -6; + def months = [] + while(add<0){ + months << DateUtils.addMonth(new Date(), add) + add++ + } + def prices = exportRegion(city, months) + def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM") + ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据") + } + + + def exportRegion(city, months){ + def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; + def datas =[] + def header = ["城市", "区域"] + for(def m : months){ + header << m.toString().substring(0, m.toString().lastIndexOf("-")) + } + datas << header + for(def r : regions){ + def prices = findRegionPrice(city, r, months); + datas << prices + } + + return datas; + } + + + + public def findRegionPrice(city, region, weekends){ + def prices = [] + prices << city + prices << region + for(def weekend : weekends){ + def startDate = DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd") + def endDate = DateUtils.getLastDayOfMonth(startDate) + def avgPrice = queryCityRegionPrice(city, region, startDate, endDate); + prices << avgPrice + } + return prices; + } + + public def queryCityRegionPrice(city, region, startDate, endDate){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("region", region); + queryCondition.put("block", null); + queryCondition.put('weekend', new BasicDBObject('$gte', startDate).append('$lte',endDate)); + DBCursor cur = REGION_STAT_COLL.find(queryCondition).sort(new BasicDBObject().append("weekend", -1)) + cur.setOptions(Bytes.QUERYOPTION_NOTIMEOUT) + def price = 0; + while(cur.hasNext()) { + DBObject dbo = cur.next() + price = dbo.get("avg_price"); + if(price){ + break; + } + } + + return price; + } + + + + public def findWeekends(limit){ + def index = 1; + def weekends = []; + def date = new Date(); + while(index <= limit){ + def time = DateUtils.getFirstDayOfWeek(date) + date = time; + weekends << DateUtils.formatDateByPattern(time, "yyyy-MM-dd"); + index ++; + } + + weekends = weekends.reverse(); + return weekends; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 05ed58021..fab36e220 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -63,8 +63,7 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = CommonDatautils.findAllCity(); - citys.remove("上海") - citys.remove("北京") + citys = citys[22..(citys.size()-1)] for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) // dists = dists[0..20] @@ -367,7 +366,7 @@ class DistrictIndexBatch extends BaseBatch{ def findHouse(def dist, def startDate, def endDate){ def nameQuery = getAllName(dist); final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT area FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1") sqlBuffer.append(" and city = '"+dist.city+"'") sqlBuffer.append(" and region = '"+dist.region+"'") sqlBuffer.append(" and cat = 'sell'") @@ -375,10 +374,12 @@ class DistrictIndexBatch extends BaseBatch{ sqlBuffer.append(" and (" + nameQuery + ")"); ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) def areas = [] + def avg_area = null while (rs.next()){ - areas << rs.getDouble("area") +// areas << rs.getDouble("area") + avg_area = rs.getDouble("avg_area") } - return areas; + return avg_area; } def getAllName(dist){ From dfb71625f1ef25727570f09d95e6915a6e8b5bd4 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Feb 2022 14:02:03 +0800 Subject: [PATCH 172/212] =?UTF-8?q?=E7=9B=B8=E5=85=B3=E8=84=9A=E6=9C=AC?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/CcbFtpUtils.groovy | 4 +- .../ccbevaluation/ExportOracleData.groovy | 64 ++-- .../data/MatchDealPriceRegion.groovy | 7 +- .../ronghe/ExportDistrictRongHe.groovy | 315 ++++++++++++------ .../script/commData/CommonDatautils.groovy | 12 + .../script/fuzhou/FzDataResultStat.groovy | 32 +- .../script/gj/data/TempMatchDistrict.groovy | 70 ++++ .../script/gj/level/DistrictIndexBatch.groovy | 12 +- 8 files changed, 376 insertions(+), 140 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy index 62ace6306..3fffa870c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpUtils.groovy @@ -19,10 +19,10 @@ class CcbFtpUtils { static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); - static def ftpUpload(outputPath_dist, month){ + static def ftpUpload(type,outputPath_dist, month){ //上传到ftp sftpChannel = reConnect(); - if(month.toString().contains("10/") || month.toString().endsWith("10")){ + if(type && type=="rh"){ sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel); }else{ sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index cb4fdc1e5..1bd63cbeb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -1,6 +1,7 @@ package com.fangjia.script.ccbevaluation import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.ccbevaluation.CcbFtpUtils import com.fangjia.script.gj.ccb.FileUtils @@ -37,9 +38,9 @@ class ExportOracleData { // exportCommunityStaging() // exportCommunityBuileType() - exportThirdPartyPrice2(lastMonth) +// exportThirdPartyPrice2(lastMonth) // exportBigdataForsale("2021-12-19") - exportBigdataDeal("2021-12-01") +// exportBigdataDeal("2021-12-01") //计算结果相关 @@ -54,9 +55,12 @@ class ExportOracleData { // // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) -// -// //ok文件 -// exportLastFile() + + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); + //ok文件 + exportLastFile() } @@ -85,7 +89,7 @@ class ExportOracleData { def headers = []//["系数ID","系数类型","系数","排序","城市","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "上传Ftp完成......" } @@ -120,7 +124,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_POI_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_POI_GJ 上传Ftp完成......" } @@ -147,7 +151,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "COMMUNITY_STAGING_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "COMMUNITY_STAGING_GJ 上传Ftp完成......" } @@ -181,7 +185,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_COMMBUILDTYPE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_COMMBUILDTYPE_GJ 上传Ftp完成......" } @@ -218,7 +222,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_DISTRICT_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_DISTRICT_GJ 上传Ftp完成......" } @@ -253,7 +257,7 @@ class ExportOracleData { //["第三方均价ID","楼盘ID","小区名称","城市ID","城市名称","行政区ID","行政区名称","指导日期","指导价格","添加人","备注","创建时间","更新时间","来源公司"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "GUIDED_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "GUIDED_PRICE_GJ 上传Ftp完成......" } @@ -286,7 +290,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "COMMUNITY_GUIDE_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "COMMUNITY_GUIDE_PRICE_GJ 上传Ftp完成......" } @@ -376,7 +380,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_DEAL_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_DEAL_GJ 上传Ftp完成......" } @@ -425,7 +429,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "THIRD_PARTY_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "THIRD_PARTY_PRICE_GJ 上传数据文件完成......" } @@ -460,7 +464,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_THIRD_PARTY_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) } @@ -514,7 +518,7 @@ class ExportOracleData { //"乡镇/街道办","逻辑行政区划代码","逻辑行政区划","状态"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "BIGDATA_COMMUNITY_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_COMMUNITY_GJ 上传Ftp完成......" } @@ -606,7 +610,7 @@ class ExportOracleData { } println "BIGDATA_FORSALE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "BIGDATA_FORSALE_GJ 上传Ftp完成......" } @@ -669,8 +673,8 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "AVM_EVALUATION_GJNB 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) - CcbFtpUtils.ftpUpload(fileName1, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) println "AVM_EVALUATION_GJNB 上传Ftp完成......" } @@ -719,8 +723,8 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) - CcbFtpUtils.ftpUpload(fileName1, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" } @@ -762,8 +766,8 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) - CcbFtpUtils.ftpUpload(fileName1, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" } @@ -802,7 +806,7 @@ class ExportOracleData { def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" } @@ -831,7 +835,7 @@ class ExportOracleData { def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "CITY_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "CITY_PRICE_GJ 上传Ftp完成......" } @@ -862,7 +866,7 @@ class ExportOracleData { def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); println "DISTRICT_PRICE_GJ 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "DISTRICT_PRICE_GJ 上传Ftp完成......" } @@ -930,7 +934,7 @@ class ExportOracleData { FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); // } println "community_p_describe_gj 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "community_p_describe_gj 上传Ftp完成......" } @@ -967,12 +971,10 @@ class ExportOracleData { if(allRows.size()>0){ FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); } - - println city } println "comm_case_detail_gj 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName, curDate) println "comm_case_detail_gj 上传Ftp完成......" } @@ -983,7 +985,7 @@ class ExportOracleData { dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" } FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) + CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate) } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy index c52ecb26b..167fda35a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchDealPriceRegion.groovy @@ -31,13 +31,16 @@ class MatchDealPriceRegion { if(!newRegion){ index++ newRegion = findRegionByThirdPrice(d['B'], d['D']); - if(!newRegion || (newRegion && newRegion==d['C'])){ + if(!newRegion){ def rets = CommonDatautils.findDistrictsByName(d['B'], d['D'],"district") + if(!rets){ + rets = CommonDatautils.findDistrictByAlias(d['B'], null,d['D'],"district") + } if(rets && rets.size()==1){ newRegion = rets[0].region } } - if(newRegion && newRegion!=d['C']){ + if(newRegion){ def query = new BasicDBObject() query.put("_id", new ObjectId(d['A'])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index a027b584e..fff8d13cf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -24,43 +24,77 @@ class ExportDistrictRongHe extends BaseBatch{ @Override void process(Object args) { - def citys = CcbCommonUtils.citys; - def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + //按天导(每天一个文件夹) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) + if(curDate.toString().endsWith("10")){ + exportBigdataCommunityGJRH() + exportBigdataBuildingFJW() + exportBigdataCommfeeGj() -// //检查ok文件是否上传 -// exportAllBigdataDealFJW() - for(def city : citys) { - def cityCode = cityCodeDataMap.get(city) -// -// exportBigdataDealFJW(cityCode, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) -// - exportBigdataCommunityGJRH(cityCode) - exportBigdataBuildingFJW(cityCode) - exportBigdataCommfeeGj(cityCode) -// - exportBigdataCommbuildstruGJ(cityCode) - exportBigdataCommhousetypeGJ(cityCode) - exportBigdataCommbuildyearGj(cityCode) - exportBigdataCommhouseuseGJ(cityCode) - exportBigdataPositionGJ(cityCode) - exportBigdataDistrictFJW(cityCode) - exportBigdataAliasFJW(cityCode) + exportBigdataCommbuildstruGJ() + exportBigdataCommhousetypeGJ() + exportBigdataCommbuildyearGj() + exportBigdataCommhouseuseGJ() + exportBigdataPositionGJ() + exportBigdataDistrictFJW() + exportBigdataAliasFJW() - exportBigdataBuildprojFjw(cityCode) - exportBigdataCommlayoutpicFjw(cityCode) + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() } - exportEvaluateRate() - exportOk() -// -// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/package.sh ${curDate}")); + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); + + exportOk(curDate) } +// +// def runByCity(){ +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// +// +//// //检查ok文件是否上传 +//// exportAllBigdataDealFJW() +//// for(def city : citys) { +//// def cityCode = cityCodeDataMap.get(city) +//// +// //按天导(一个文件夹下) +// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) +// //按天导(每天一个文件夹) +// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) +// +//// exportBigdataCommunityGJRH(cityCode) +//// exportBigdataBuildingFJW(cityCode) +//// exportBigdataCommfeeGj(cityCode) +//// +//// exportBigdataCommbuildstruGJ(cityCode) +//// exportBigdataCommhousetypeGJ(cityCode) +//// exportBigdataCommbuildyearGj(cityCode) +//// exportBigdataCommhouseuseGJ(cityCode) +//// exportBigdataPositionGJ(cityCode) +//// exportBigdataDistrictFJW(cityCode) +//// exportBigdataAliasFJW(cityCode) +//// +//// exportBigdataBuildprojFjw(cityCode) +//// exportBigdataCommlayoutpicFjw(cityCode) +//// } +//// exportEvaluateRate() +// +//// +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// +// exportOk() +// } + def exportBigdataCommunityGJRH(cityCode){ - def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW where CITYID='${cityCode}'"; + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -86,21 +120,18 @@ class ExportDistrictRongHe extends BaseBatch{ data << r.SOURCECODE allRows << data }); - - - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" } - def exportBigdataCommbuildstruGJ(cityCode){ - def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW where CITYID='${cityCode}'"; + def exportBigdataCommbuildstruGJ(){ + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -114,17 +145,17 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" } def exportBigdataCommhousetypeGJ(cityCode){ - def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW where CITYID='${cityCode}'"; + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -138,17 +169,17 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......" } - def exportBigdataCommbuildyearGj(cityCode){ - def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW where CITYID='${cityCode}'"; + def exportBigdataCommbuildyearGj(){ + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -162,17 +193,17 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......" } - def exportBigdataCommfeeGj(cityCode){ - def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW where CITYID='${cityCode}'"; + def exportBigdataCommfeeGj(){ + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -196,18 +227,18 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" } - def exportBigdataCommhouseuseGJ(cityCode){ - def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW where CITYID='${cityCode}'"; + def exportBigdataCommhouseuseGJ(){ + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -222,17 +253,17 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......" } - def exportBigdataPositionGJ(cityCode){ - def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW where CITYID='${cityCode}'"; + def exportBigdataPositionGJ(){ + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -251,17 +282,17 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_POSITION_FJW 上传Ftp完成......" } - def exportBigdataDistrictFJW(cityCode){ - def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW where CITYID='${cityCode}'"; + def exportBigdataDistrictFJW(){ + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -279,17 +310,17 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" } - def exportBigdataAliasFJW(cityCode){ - def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW where city_id='${cityCode}'"; + def exportBigdataAliasFJW(){ + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}' def allRows = [] sqlInstance.eachRow(sql,{r-> def data = [] @@ -310,16 +341,16 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "ADDRESS_ALIAS_FJW 上传Ftp完成......" } - def exportBigdataDealFJW(cityCode, dealdate){ + def exportBigdataDealFJW(dealdate){ def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") @@ -334,14 +365,14 @@ class ExportDistrictRongHe extends BaseBatch{ index++ def curStartStr = curStartDateStr - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" } def allRows = [] if(index==1){ - def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where CITYID='${cityCode}' and dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')"; + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and sqlInstance.eachRow(sql,{r-> def data = [] data << r.CITYID @@ -420,13 +451,107 @@ class ExportDistrictRongHe extends BaseBatch{ stop = true } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh", fileName, curDate) println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" def curEndStr = DateUtils.add(curStartStr, 1) curStartDateStr = curEndStr } } + + def exportBigdataDealFJW2(curDate, dealdate){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + def allRows = [] + if(curDate.toString().endsWith("01")){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + } + def exportAllBigdataDealFJW(){ def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE def data = [] @@ -547,12 +672,12 @@ class ExportDistrictRongHe extends BaseBatch{ allRows << data }); def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_BUILDING_FJW 上传Ftp完成......" } @@ -580,40 +705,40 @@ class ExportDistrictRongHe extends BaseBatch{ fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate) + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "EVALUATE_RATE_FJW 上传Ftp完成......" } - def exportBigdataBuildprojFjw(cityCode){ + def exportBigdataBuildprojFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" } - def exportBigdataCommlayoutpicFjw(cityCode){ + def exportBigdataCommlayoutpicFjw(){ def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" - def fileName = "/ccb_rh/${curDate}/${cityCode}/FJW_000000_${cityCode}_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${cityCode}\\FJW_000000_${cityCode}_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" } FileUtils.batchCreateFile([], fileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(fileName, curDate+"/${cityCode}") + CcbFtpUtils.ftpUpload("rh",fileName, curDate) println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" } - def exportOk(){ - def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def exportOk(curDate){ +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" if(CommonDatautils.isWindow()){ dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" } FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); - CcbFtpUtils.ftpUpload(dataCodeFileName, curDate) + CcbFtpUtils.ftpUpload("rh",dataCodeFileName, curDate) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 63732e34f..9502023d9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -1356,6 +1356,18 @@ class CommonDatautils { } return regions; } + + public static def getOrgName(def city, def name){ + DBObject queryCondition = new BasicDBObject() + queryCondition.put("city", city); + queryCondition.put("name", name); + def dbCursor = REGION_BLOCK_COLL.find(queryCondition) + def orgName = null + for(DBObject obj : dbCursor){ + orgName = obj.org_name + } + return orgName; + } public static def getAllCityRegions(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 46ad15c1b..072e16e0a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -19,8 +19,8 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() - //task.priceMarkupRun(); - task.regionPriceTrendRun() + task.priceMarkupRun(); +// task.regionPriceTrendRun() } def priceMarkupRun(){ def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); @@ -42,20 +42,44 @@ class FzDataResultStat { map2.put(key, row[5]) } def rows = [] + rows << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] map1.each {k,v-> def row = [] def keys = k.split("##") def lastPrice = map2.get(k) def markup = null if(v && lastPrice){ - markup = (v - lastPrice)*100.0/lastPrice + markup = (v.toInteger() - lastPrice.toInteger())*100.0/lastPrice.toInteger() } rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup] } - ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") + //ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; + def datas1 = [] + def datas2 = [] + datas1 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] + datas2 << ["城市","区域","名称","${curMonth}价格","${lastMonth}价格", "涨跌幅"] + def row_num = 0 + for(def row : rows){ + if(row_num>0){ + def region = row[1] + if(regions.contains(region)){ + if(row[5] && Math.abs(row[5])>10){ + datas1 << row + } + }else{ + if(row[5] && Math.abs(row[5])>10){ + datas2 << row + } + } + } + row_num++ + } + + ExcelUtils.write("D:\\数据\\福州\\6区(超10%)_${curMonth}.xlsx", datas1, "数据") + ExcelUtils.write("D:\\数据\\福州\\7县(超10%)_${curMonth}.xlsx", datas2, "数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy new file mode 100644 index 000000000..9b1c8cfc3 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -0,0 +1,70 @@ +package com.fangjia.script.gj.data + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import org.apache.xmlbeans.impl.regex.Match + +import java.util.regex.Pattern + +class TempMatchDistrict { + static main(args) { + def task = new TempMatchDistrict() +// task.run() + task.regionConver() + } + + def regionConver(){ + //,"北京" + def files = ["广州","杭州","合肥","上海","深圳","石家庄","苏州","天津","武汉","西安","重庆"] + for(def f : files){ + ExcelUtils.multiThreadingModify("D:\\数据\\其它\\均价数据\\"+f+".xlsx",{d-> + def orgName = CommonDatautils.getOrgName(f, d['M']) + if(orgName){ + d['M'] = orgName + } + }) + } + } + def run(){ + def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据" + def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"] + for(def f : files){ + def filePath = path+"\\"+f; + match(filePath); + } + } + def match(filePath){ + ExcelUtils.multiThreadingModify(filePath,{d-> + def city = d['B'].replace("市","") +// def newRegion = RegionData.formatRegion(city, d['C']) +// if(newRegion){ + def rMap = SeaweedDataMatchProcess.doMatch(city, d['C'], d['D'], null); + if(rMap){ + def dist = CommonDatautils.queryDistrictById(rMap.id) + if(dist){ + d['N'] = dist._id.toString() + d['O'] = CommonDatautils.getOrgName(city, dist.region) + d['P'] = dist.name + d['Q'] = dist.lng2/10000000000.0 + d['R'] = dist.lat2/10000000000.0 + d['S'] = getYear(dist.finish_building_date) + } + } +// } + }) + } + + + def getYear(finishBuildingDate){ + if(finishBuildingDate){ + Pattern pat3 = Pattern.compile("\\d{4}"); + def matcher3 = pat3.matcher(finishBuildingDate); + if(matcher3.find()){ + return matcher3.group() + } + } + return null + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index fab36e220..67160f9e4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -164,12 +164,12 @@ class DistrictIndexBatch extends BaseBatch{ def getHousePrice(d, dists, priceMap, citycode){ //def areas = findHouse(d, 20200101, 20220101) - def areas = [] - if(citycode){ - areas = findHouseByOracle(d._id.toString(), citycode); - }else{ - areas = findHouse(d, 20200101, 20220101) - } + def areas = [] +// if(citycode){ +// areas = findHouseByOracle(d._id.toString(), citycode); +// }else{ + areas = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) +// } def avg_area = null if(areas){ avg_area = areas.sum()/areas.size(); From 3da0be46d7aac16b5e3727a4df88c49676c90015 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Jan 2022 16:59:03 +0800 Subject: [PATCH 173/212] =?UTF-8?q?=E6=BD=9C=E5=8A=9B=E6=8C=87=E6=95=B0?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/level/DistrictIndexBatch.groovy | 145 +++++++++++------- 1 file changed, 87 insertions(+), 58 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 67160f9e4..22d0bcd03 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -63,7 +63,7 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = CommonDatautils.findAllCity(); - citys = citys[22..(citys.size()-1)] +// citys = citys[22..(citys.size()-1)] for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) // dists = dists[0..20] @@ -157,75 +157,76 @@ class DistrictIndexBatch extends BaseBatch{ fileName = "D:\\数据\\评级\\${city}综合潜力.dat" } - FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", true); + FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") } def getHousePrice(d, dists, priceMap, citycode){ //def areas = findHouse(d, 20200101, 20220101) - def areas = [] +// def areas = [] // if(citycode){ // areas = findHouseByOracle(d._id.toString(), citycode); // }else{ - areas = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) +// areas = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) // } - def avg_area = null - if(areas){ - avg_area = areas.sum()/areas.size(); - }else{ - if(d.finish_building_date){ - def year = CommonUtils.getBuildYear(d.finish_building_date) - if(year){ - def startYear = (year.toString().substring(0,3)+"0").toInteger() - def endYear = startYear + 10 - - def selectDists = [] - for(def ad : dists){ - if(d.region == ad.region){ - def adYear = CommonUtils.getBuildYear(ad.finish_building_date) - if(adYear){ - adYear = adYear.toInteger() - if(adYear>=startYear && adYear<=endYear){ - selectDists << ad - } - } - } - } - - if(selectDists){ - println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size() - def allAreas = [] - def ids = [] - if(citycode){ - for(def sd : selectDists){ - ids << sd._id.toString() - } - if(ids.size()>=901){ - ids = ids[0..900] - } - - def idstr = ids.join("','") - allAreas = findHouseByOracle2(idstr,citycode); - }else{ - for(def sd : selectDists){ - def areaDatas = findHouse(sd, 20190101, 20220101) - allAreas.addAll(areaDatas) - } - } - if(allAreas){ - avg_area = allAreas.sum()/allAreas.size(); - } - } - } - } + def avg_area = findHouse(d, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) + if(!avg_area){ + avg_area = findHousByRegion(d.city, d.region, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) +// if(d.finish_building_date){ +// def year = CommonUtils.getBuildYear(d.finish_building_date) +// if(year){ +// def startYear = (year.toString().substring(0,3)+"0").toInteger() +// def endYear = startYear + 10 +// +// def selectDists = [] +// for(def ad : dists){ +// if(d.region == ad.region){ +// def adYear = CommonUtils.getBuildYear(ad.finish_building_date) +// if(adYear){ +// adYear = adYear.toInteger() +// if(adYear>=startYear && adYear<=endYear){ +// selectDists << ad +// } +// } +// } +// } +// +// if(selectDists){ +// println d.region+" "+d.name+" 周边同建筑年代楼盘:"+selectDists.size() +// def allAreas = [] +// def ids = [] +// if(citycode){ +// for(def sd : selectDists){ +// ids << sd._id.toString() +// } +// if(ids.size()>=901){ +// ids = ids[0..900] +// } +// +// def idstr = ids.join("','") +// allAreas = findHouseByOracle2(idstr,citycode); +// }else{ +// for(def sd : selectDists){ +// def areaDatas = findHouse(sd, 20190101, 20220101) +// allAreas.addAll(areaDatas) +// } +// } +// if(allAreas){ +// avg_area = allAreas.sum()/allAreas.size(); +// } +// } +// } +// } } if(avg_area){ def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) - def total_price = (avg_price * avg_area)/10000.0 - def total = DataHandleUtils.getDoubleRound(2, total_price) - return total + if(avg_price){ + def total_price = (avg_price * avg_area)/10000.0 + def total = DataHandleUtils.getDoubleRound(2, total_price) + return total + } } return null } @@ -236,7 +237,7 @@ class DistrictIndexBatch extends BaseBatch{ for(def i=0;i=3){ break; @@ -287,7 +288,13 @@ class DistrictIndexBatch extends BaseBatch{ //排序 Collections.sort(list, new Comparator() { @Override - int compare(Object o1, Object o2) {return o1.distance-o2.distance;} + int compare(Object o1, Object o2) { + if(o1.distance > o2.distance) + return 1;//第二个元素(o1)比第一个元素(o2)大,返回-1 + if(o1.distance == o2.distance) + return 0; + return -1; + } }) return list; } @@ -379,6 +386,28 @@ class DistrictIndexBatch extends BaseBatch{ // areas << rs.getDouble("area") avg_area = rs.getDouble("avg_area") } + if(avg_area){ + avg_area = DataHandleUtils.getDoubleRound(2, avg_area) + } + return avg_area; + } + + + def findHousByRegion(def city, def region, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT avg(area) as avg_area FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def avg_area = null + while (rs.next()){ + avg_area = rs.getDouble("avg_area") + } + if(avg_area){ + avg_area = DataHandleUtils.getDoubleRound(2, avg_area) + } return avg_area; } From c823e8bfb244a832f557adfd808984e3fdcb3faf Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 25 Jan 2022 17:53:18 +0800 Subject: [PATCH 174/212] =?UTF-8?q?=E5=9F=8E=E5=B8=82=E6=88=B7=E5=9D=87?= =?UTF-8?q?=E4=BB=B7=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/gj/level/DistrictIndexBatch.groovy | 39 ++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 22d0bcd03..5547c7ce9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -63,16 +63,28 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = CommonDatautils.findAllCity(); -// citys = citys[22..(citys.size()-1)] + citys.remove("上海") for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) -// dists = dists[0..20] def code = cityCodeDatas.get(city) - calcute(city, dists, code) + def city_avg_total_price = null + if(city!="上海"){ + city_avg_total_price = getAvgTotalPrice(city, 20200101, DateUtils.formatDateByPattern(new Date(),"yyyyMMdd")) + if(city_avg_total_price){ + city_avg_total_price = city_avg_total_price/10000.0 + city_avg_total_price = DataHandleUtils.getDoubleRound(6, city_avg_total_price) + } + }else{ + city_avg_total_price = 531.2615913 + } + if(city_avg_total_price){ + calcute(city, dists, code, city_avg_total_price) + } + println city +" "+city_avg_total_price } } - def calcute(city, dists, citycode){ + def calcute(city, dists, citycode, city_avg_total_price){ def weekend = DateUtils.getFirstDayOfWeek(new Date()); def priceMap = CommonDatautils.queryDistrictPrice(city, weekend) def costDatas = getCostByCity(city) @@ -100,7 +112,7 @@ class DistrictIndexBatch extends BaseBatch{ def total_price_index = 0 if (total_price) { - total_price_index = 286.0134196 * (total_price - 531.2615913) / 313.3693508 + total_price_index = 286.0134196 * (total_price - city_avg_total_price) / 313.3693508 total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index) } def avg_charge_index = 0 @@ -161,6 +173,23 @@ class DistrictIndexBatch extends BaseBatch{ // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") } + def getAvgTotalPrice(city, startDate, endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT avg(total_price) as avg_total_price FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def avg_total_price = null + while (rs.next()){ + avg_total_price = rs.getDouble("avg_total_price") + } + if(avg_total_price){ + avg_total_price = avg_total_price.toInteger() + } + return avg_total_price; + } + def getHousePrice(d, dists, priceMap, citycode){ //def areas = findHouse(d, 20200101, 20220101) From 12c1b22b6de62699f637ac3cc17b89889c1a0eee Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Wed, 26 Jan 2022 14:24:17 +0800 Subject: [PATCH 175/212] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=B8=A4=E6=B5=B7?= =?UTF-8?q?=E9=93=B6=E8=A1=8C=E4=BC=B0=E4=BB=B7=E5=A4=B1=E8=B4=A5=E5=BC=82?= =?UTF-8?q?=E5=B8=B8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../controllers/weixin/WeiXinEvaluationController.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java index 3c0a6bc47..272979fca 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/weixin/WeiXinEvaluationController.java @@ -462,8 +462,14 @@ public class WeiXinEvaluationController { }else if(bhDists.size()>1){ int mod = Integer.MAX_VALUE; for(JSONObject jo : bhDists){ - double price = Double.valueOf((String)jo.get("price")); - int bhprice = (int)price; + int bhprice = 0; + if (jo.get("price") == null || jo.get("price").equals("null")) { + bhprice = 0; + } else { + double price = Double.valueOf((String)jo.get("price")); + bhprice = (int)price; + } + int bhmod = Math.abs(bhprice - avgPrice); if(bhmod Date: Mon, 21 Feb 2022 11:36:37 +0800 Subject: [PATCH 176/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=89=8D=E6=B5=B7?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../temp/shenjian/qianhai/QianhaiTest.groovy | 89 +++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy new file mode 100644 index 000000000..b0b4e16c2 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.temp.shenjian.qianhai + +import com.fangjia.core.util.HttpRequest +import com.fangjia.script.util.CommonUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.OpenApiClient +import com.fangjia.script.xingyebank.CityUtil +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据test.xlsx",{List item-> + + if(StringUtils.isBlank(item['B'])){ + return; + } + HttpRequest request = new HttpRequest() + def params = [:] + params["address"] = item['B'] + params["city"] = getCityFormAddress(item['B']) + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + JSONObject json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + + if(json.code == 200 && json.result){ + JSONObject result = json.result + item['C'] = CityUtil.getProvinceByCity(result.city) + item['D'] = result.city + item['E'] = result.district + item['F'] = result.name + if (result.name) { + item['O'] = '住宅地址' + } else { + item['O'] = '非住宅地址' + } + item['G'] = result.address + item['H'] = result.lng + item['I'] = result.lat + + // 小区均价查询 + int avgPrice = OpenApiClient.queryDistrictAvgPrice(result.city,result.district,result.name) + if(avgPrice > 0){ + item['M'] = avgPrice + } + params.clear() + params["city"] = result.city + params["district"] = result.district + params["name"] = result.name + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + // 小区周边一公里均价 + json = request.getJsonContent("http://open.fangjia.com/property/aroundAverage", params, "UTF-8") + if(json.result){ + item['J'] = json.getJSONObject("result").averagePrice + } + } + params.clear() + params["city"] = getCityFormAddress(item['B']) + params["address"] = item['B'] + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + + // 地址认证接口 + json = request.getJsonContent("http://open.fangjia.com/address/realAuth", params, "UTF-8") + if(json && json.result){ + JSONObject result = json.result + item['K'] = result.formattedAddress + item['L'] = result.authLevel + } + // 小区详情接口 + params.clear() + params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" + params["city"] = getCityFormAddress(item['B']) + params["name"] = item['F'] + params["region"] = item['E'] + + json = request.getJsonContent("http://open.fangjia.com/district/show", params, "UTF-8") + if(json){ + item['N'] = json.buildType + } + + println(item) +}) + +def getCityFormAddress(String addr){ + List cities = CommonUtils.findAllCity() + for(String city : cities){ + if(addr.contains(city)){ + return city + } + } + return null +} \ No newline at end of file From 72237096e81ad247b1b515f21e987032e936195e Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 21 Feb 2022 11:49:50 +0800 Subject: [PATCH 177/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=89=8D=E6=B5=B7?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy index b0b4e16c2..7c92e5a3e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy @@ -57,7 +57,7 @@ ExcelUtils.multiThreadingModify("D:\\ params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" // 地址认证接口 - json = request.getJsonContent("http://open.fangjia.com/address/realAuth", params, "UTF-8") + json = request.getJsonContent("http://open.fangjia.com/address/auth", params, "UTF-8") if(json && json.result){ JSONObject result = json.result item['K'] = result.formattedAddress From 2cea6c4190c3db222915d7c3a14cfdc3d0c6b43a Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Mon, 21 Feb 2022 11:56:06 +0800 Subject: [PATCH 178/212] =?UTF-8?q?=E4=BC=98=E5=8C=96=E5=89=8D=E6=B5=B7?= =?UTF-8?q?=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/temp/shenjian/qianhai/QianhaiTest.groovy | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy index 7c92e5a3e..6994cf1dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/qianhai/QianhaiTest.groovy @@ -8,7 +8,7 @@ import com.fangjia.script.xingyebank.CityUtil import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据test.xlsx",{List item-> +ExcelUtils.multiThreadingModify("D:\\数据\\前海\\前海征信测试数据.xlsx",{List item-> if(StringUtils.isBlank(item['B'])){ return; @@ -20,6 +20,11 @@ ExcelUtils.multiThreadingModify("D:\\ params["token"] = "F54F52381C49BB9EB4A33EB1B65604AE4B71A28AEE53518A94A2F360408B9056D57553931D15CE6DDE765562DAD286BA38E05A4CDAFC51C3D527A4959BF8E75A3B95DB7108FCEA340DDE61925616DB55118E1851E67D83EAD800460D100D6B667A4ED8EE67C8F7FB" JSONObject json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + if((!json || !json.result) && StringUtils.isNotBlank(item['G'])){ + params["address"] = item['G'] + json = request.getJsonContent("http://open.fangjia.com/qhzx/propertyDetail", params, "UTF-8") + } + if(json.code == 200 && json.result){ JSONObject result = json.result item['C'] = CityUtil.getProvinceByCity(result.city) @@ -28,8 +33,6 @@ ExcelUtils.multiThreadingModify("D:\\ item['F'] = result.name if (result.name) { item['O'] = '住宅地址' - } else { - item['O'] = '非住宅地址' } item['G'] = result.address item['H'] = result.lng @@ -51,6 +54,9 @@ ExcelUtils.multiThreadingModify("D:\\ item['J'] = json.getJSONObject("result").averagePrice } } + if (!item['O']) { + item['O'] = '非住宅地址' + } params.clear() params["city"] = getCityFormAddress(item['B']) params["address"] = item['B'] From 5b562f22dfd8c9ce91a5e7b8c815f97b22ce3217 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Feb 2022 10:27:55 +0800 Subject: [PATCH 179/212] =?UTF-8?q?=E6=88=BF=E6=BA=90=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../script/gj/district/ZsNoExistsToDB.groovy | 340 ++++++++++++++++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 383 ++++++++++++++++++ 3 files changed, 724 insertions(+), 1 deletion(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index 7138e276c..b715ccdde 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20211218, 20220119); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy new file mode 100644 index 000000000..ce6949835 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy @@ -0,0 +1,340 @@ +package com.fangjia.script.gj.district + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import com.fangjia.core.util.BaiDuApiUtils + +import java.sql.ResultSet + +class ZsNoExistsToDB { + + public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" + public static EsDataSource dataSource = new EsDataSource() + static { + dataSource.setUrl(address) + Properties connectionProperties = new Properties() + connectionProperties.put("user", "developer") + connectionProperties.put("password", "5e8edd851d2fdfbd7415232c67367cc3") + dataSource.setProperties(connectionProperties) + } + + + static main(args) { + def task = new ZsNoExistsToDB() + task.run() +// task.addDistrictToDB() + } + + def addDistrictToDB(){ + def datas = ExcelUtils.read("D:\\数据\\浙商\\不存在小区(结果V1).xlsx",0,1) + def keys = [] + def addIndex = 0 + for(def d : datas){ + if(d['J'] && d['J']=="新增"){ + def city = d['A'] + def region = d['B'] + def name = d['F'].toString().replace("·","") + def address = d['G'] + def dist = CommonDatautils.findDistrict(city,region,name,"district") + if(!dist){ + dist = CommonDatautils.findDistrict(city,region,d['C'],"district") + } + if(!dist){ + keys << city+"##"+region+"##"+name + def alias = [] + def str = d['H'] + if(str){ + def arrs = str.split(",") + for(def s : arrs){ + if(!alias.contains(s)){ + alias << s + } + } + } + + if(address){ + address = AddressUtil.formatAddress(city, address) + } + def lng = d['M'] + def lat = d['L'] + def lng2 = null + def lat2 = null + long lng_new = 0 + long lat_new = 0 + long lng2_new = 0 + long lat2_new = 0 + if(lng){ + lng = lng.toString().toDouble() + lat = lat.toString().toDouble() + // 高德转百度 + def baidu = BaiDuApiUtils.geoconv(lng, lat, 3) + lng2 = baidu.x + lat2 = baidu.y + lng_new = lng *10000000000L + lat_new = lat *10000000000L + if(lng2){ + lng2_new = lng2 *10000000000L + lat2_new = lat2 *10000000000L + } + } + + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + } + + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", city) + data.put("region",region) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", address) + if(lng_new){ + data.put("lat",lat_new) + data.put("lng",lng_new) + data.put("lat2",lat2_new) + data.put("lng2",lng2_new) + } + if(alias){ + data.put("alias",alias) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + DbUtils.seaweed.save(data) + FangjiaClientUtils.sendDistrictPriceCalculate(city,region, name, 12); + addIndex++ + println "[新增] "+addIndex +" "+ data + + }else{ + println dist.city +" "+dist.region+" "+dist.name+" 小区已存在" + } + } + } + } + + def run(){ + def rets = findOpenLog2() + ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据") + println rets.size() + +// matchMap2() + } + + def matchMap2() { + ExcelUtils.multiThreadingModify("D:\\数据\\浙商\\不存在小区.xlsx", { d -> + if(!d['E']){ + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject(d['A'],d['B'],d['C'], rets); + + def locationMap = [:] + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); + map = findStandardObject(d['A'],d['B'],d['C'], rets); + } + if(map){ + println map + if(map.adname){ + d['E'] = map.adname + }else if(map.region){ + d['E'] = map.region + } + d['F'] = map.name + d['G'] = map.address + d['K'] = map.estateType + d['L'] = map.lat + d['M'] = map.lng + d['N'] = map.lat2 + d['O'] = map.lng2 + d['P'] = map.source + d['Q'] = RegionData.formatRegion(d['A'], d['E']) + } + } + }) + } + + def findStandardObject(def city, def region, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + for (def mapObj : rets) { +// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){ + mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address); + map = mapObj; + break; + }else if(mapObj.address && mapObj.address.contains(name)){ + map = mapObj; + break; + }else if(mapObj.name && name){ + double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name); + if(score1){ + map = mapObj; + break; + } + } +// } + } + } + } + return map; + } + + + def matchMap(){ + ExcelUtils.modify("D:\\数据\\浙商\\不存在小区.xlsx",{data-> + def gdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "高德") + if(!gdData){ + gdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "高德") + } + if(gdData){ + data['E'] = gdData.district + data['F'] = gdData.name + data['G'] = gdData.address + data['H'] = gdData.type + data['I'] = gdData.lng + data['J'] = gdData.lat + data['K'] = gdData.score + } + + def bdData = FangjiaClientUtils.getMapData(data['A'], data['B'], data['C'], data['D'],"", "百度") + if(!bdData){ + bdData = FangjiaClientUtils.getMapData(data['A'], "", data['C'], data['D'],"", "百度") + } + if(bdData){ + data['L'] = bdData.district + data['M'] = bdData.name + data['N'] = bdData.address + data['O'] = bdData.type + data['P'] = bdData.lng + data['Q'] = bdData.lat + data['R'] = bdData.score + } + },0) + } + + + def findOpenLog2(){ + def time = DateUtils.parseFromTightYYYYMMDD("2022-01-01") + def endDate = DateUtils.parseFromTightYYYYMMDD("2022-02-18") + QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()).to(endDate).includeLower(true).includeUpper(false); + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007569')).must(QueryBuilders.termQuery("request_uri", '/czbank/evaluation')).must(date_range_cdate) + def rets = [] + rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"] + def datas = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def index = 0 + def count = 0 + def keys = [] + for(def d : datas){ + if(d.code == 400){ + index++ + def msg = d.msg + if(msg.contains("该楼盘不存在")){ + def arrs = msg.split(" ") + if(arrs){ + def rMap = [:] + for(def s : arrs){ + if(s.contains(":")){ + def ars = s.split(":") + if(ars.size()==2){ + rMap.put(ars[0], ars[1]) + }else{ + rMap.put(ars[0], null) + } + } + } + if(rMap){ + count++ + def key = rMap.city+"#"+rMap.region+"#"+rMap.name + if(!keys.contains(key)){ + keys << key + def params = d.parameter_map + def address = null + if(params && params.address){ + address = params.address + } + def r = [rMap.city, rMap.region, rMap.name, address] + if(!rets.contains(r)){ + rets << r + } + } + } + } + }else{ + println msg + } + } + } + println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size() + return rets; + } + + def findOpenLog(){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT * FROM open_uri_log WHERE 1=1") + sqlBuffer.append(" and user_id = '1007569'") + sqlBuffer.append(" and request_uri = '/czbank/evaluation'") + sqlBuffer.append(" and c_date >= '2022-01-01'") + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def rets = [] + rets << ["城市", "区域", "名称","地址", "高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","DB存在"] + def index = 0 + def count = 0 + while (rs.next()){ + def code = rs.getInt("code") + if(code == 400){ + index++ + def msg = rs.getString("msg") + if(msg && msg.contains("该楼盘不存在")){ + def arrs = msg.split(" ") + if(arrs){ + def rMap = [:] + for(def s : arrs){ + if(s.contains(":")){ + def ars = s.split(":") + if(ars.size()==2){ + rMap.put(ars[0], ars[1]) + }else{ + rMap.put(ars[0], null) + } + } + } + if(rMap){ + count++ + def params = rs.getObject("parameter_map") + println params + def r = [rMap.city, rMap.region, rMap.name] + if(!rets.contains(r)){ + rets << r + } + } + } + }else{ + println msg + } + } + } + println "400: "+index+" 该楼盘不存在:"+count+" 去重后:"+rets.size() + return rets; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy new file mode 100644 index 000000000..320b85cdb --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -0,0 +1,383 @@ +package com.fangjia.script.gj.paimai + +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.gj.paimai.ChromeDriverPool +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import com.mongodb.QueryOperators +import com.rabbitmq.client.impl.AMQImpl +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicLong + + +public class BeiKeCrawlByChrome { + + def DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + + static main(args) { + def task = new BeiKeCrawlByChrome(); + task.run() +// task.run1() +// task.addDealRegion(); + } + + + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def cityMap = getAllCityUrls(driver) + getLogin(driver) + crawlData(driver, cityMap) + + + } + + def addDealRegion(){ + def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") + lastMonth = lastMonth +"-01" + def q = new BasicDBObject() + q.put("region", null) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = DEAL_PRICE_COLL.find(q) + def keys = [] + for(def d : dbcursor){ + keys << ["city":d.city, "name":d.district_name] + } + for(def m : keys){ + //def list = CommonDatautils.findDistrictList(m.city, null, m.name, "district") + def list = CommonDatautils.findDistrictByAlias(m.city, null, m.name, "district") + if(list && list.size()==1){ + def dist = list[0] + + def query = new BasicDBObject() + query.put("city", m.city) + query.put("region", null) + query.put("district_name", m.name) +// query.put("source", "链家在线") + query.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern(lastMonth,"yyyy-MM-dd"))) + + def data = new BasicDBObject() + data.put("region", dist.region) + def ds = DEAL_PRICE_COLL.count(query) + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + } + + def crawlData(driver, cityMap){ + def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + cityMap.each{city, cityUrl-> + //def detailUrls = getPageUrls(driver, cityUrl) + def detailUrls = getPageDeail(driver, city, cityUrl) + println cityUrl +" 详情链接:"+detailUrls.size() + // crawlDetail(driver, DEAL_PRICE_COLL, detailUrls) + } + } + + def getPageUrls(driver, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + + def getPageDeail(driver, city, cityUrl){ + def stop = false + def page = 1 + def detailUrls = [] + while(!stop){ + def url = cityUrl + "/chengjiao/pg${page}/" + driver.get(url) + def doc = Jsoup.parse(driver.getPageSource()); + def elements = JQueryUtils.select(".listContent li", doc) + if(elements){ + for(def e : elements){ + def durl = e.select(".title a").attr("href") + def dealDate = e.select(".dealDate").text() + dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") + def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd") + if(dealDate.getTime()>=endDate.getTime()){ + detailUrls << durl + parse(city, e) + }else{ + stop = true; + break; + } + } + page++; + }else{ + stop = true; + } + } + return detailUrls + } + + def parse(city,e){ + try { + def title = e.select(".title a").text() + if (title) { + def detail = new BasicDBObject() + def arrs = title.split(" ") + detail.city = city + detail.district_name = arrs[0] + detail.region = findRegionByThirdPary(detail.city, detail.district_name) + if(detail.region){ + detail.region = RegionData.formatRegion(city, detail.region) + } + + def roomStr = arrs[1] + if (roomStr) { + if (roomStr.indexOf("室") > 0) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + } + if (roomStr.indexOf("厅") > 0 && roomStr.indexOf("室") > 0) { + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + } + } + + def areaStr = arrs[2].replace("平米", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + def price = e.select(".totalPrice .number").text() + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = e.select(".dealDate").text() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "贝壳找房" + def dbo = findTrade(detail) + if (!dbo) { + def houseInfo = e.select(".houseInfo").text() + def houseInfos = houseInfo.split("\\|") + detail.direction = houseInfos[0] + detail.fitment = houseInfos[1] + def floorStr = e.select(".positionInfo").text() + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = e.select(".title a").attr("href") + detail.create_date = new Date() + detail.m_date = new Date() + println detail + DEAL_PRICE_COLL.save(detail) + } + } + }catch(Exception ex){ + ex.printStackTrace() + println e.select(".title a").attr("href") + } + } + + + def findRegionByThirdPary(city, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("name", district_name) + q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) + def dbcursor = thirdPartyColl.find(q).sort(new BasicDBObject('s_date', -1)).limit(1) + def dbo = null; + for(def d : dbcursor){ + dbo = d + } + if(dbo){ + return dbo.region + } + return null; + } + def getContent(driver, detailUrl){ + driver.get(detailUrl) + def str = driver.getPageSource() + if(str && str.contains("登录")){ + getLogin(driver) + return getContent(driver, detailUrl) + } + return str; + } + def crawlDetail(driver,DEAL_PRICE_COLL, detailUrls){ + if(detailUrls){ + for(def i=0;i=4){ + detail.city = eles[0].text().replace("房产网","") + detail.region = eles[2].text().replace("二手房成交","") + detail.district_name = eles[eles.size()-1].text().replace("二手房成交","") + } + if(detail.district_name) { + def price = JQueryUtils.selectText(".dealTotalPrice i", doc) + if (price && price.isNumber()) { + detail.total_price = price.toInteger() * 10000 + } + def areaStr = JQueryUtils.selectText("li:contains(建筑面积)", doc).replace("建筑面积", "").replace("㎡", "") + if (areaStr) { + detail.area = areaStr.toDouble() + } + + if (detail.total_price && detail.area) { + detail.avg_price = (int) (detail.total_price / detail.area) + } + def tradeStr = JQueryUtils.selectText(".house-title span", doc).replace("成交", "").trim() + if (tradeStr) { + detail.trade_date = DateUtils.parseFromTightByPattern(tradeStr, "yyyy.MM.dd") + } + detail.source = "链家在线" + def dbo = findTrade(DEAL_PRICE_COLL, detail) + if (!dbo) { + def roomStr = JQueryUtils.selectText("li:contains(房屋户型)", doc).replace("房屋户型", "") + if (roomStr) { + detail.room = roomStr.substring(0, roomStr.indexOf("室")).toInteger() + detail.hall = roomStr.substring(roomStr.indexOf("室") + 1, roomStr.indexOf("厅")) + detail.toilet = roomStr.substring(roomStr.indexOf("卫") - 1, roomStr.indexOf("卫")) + } + detail.direction = JQueryUtils.selectText("li:contains(房屋朝向)", doc).replace("房屋朝向", "") + detail.fitment = JQueryUtils.selectText("li:contains(装修情况)", doc).replace("装修情况", "") + def floorStr = JQueryUtils.selectText("li:contains(所在楼层)", doc).replace("所在楼层", "") + if (floorStr && floorStr.contains("共") && floorStr.contains("层)")) { + floorStr = floorStr.substring(floorStr.indexOf("共") + 1, floorStr.indexOf("层)")) + if (floorStr && floorStr.isNumber()) { + detail.height = floorStr.toInteger() + } + } + detail.url = detailUrl + detail.c_date = new Date() + detail.m_date = new Date() + + DEAL_PRICE_COLL.save(detail) + } + } + return detail; + } + + + def findTrade(def detail){ + def q = new BasicDBObject() + q.put("city", detail.city) + q.put("region", detail.region) + q.put("district_name", detail.district_name) + q.put("area", detail.area) + q.put("total_price", detail.total_price) + q.put("trade_date", detail.trade_date) + q.put("source", detail.source) + def dbo = DEAL_PRICE_COLL.findOne(q) + return dbo + } + + def getAllCityUrls(driver){ + driver.get("https://www.ke.com/city/") + Thread.sleep(8000) + def cityDoc = Jsoup.parse(driver.getPageSource()); + def cityEles = JQueryUtils.select(".city_province a", cityDoc) + def cityMap = [:] + if(cityEles){ + for(def e : cityEles){ + def href = e.attr("href") +// if(e.text().contains("天津")){ + cityMap.put(e.text(), "https:"+href) +// } + } + } + return cityMap + } + + def getLogin(driver){ + driver.get("https://clogin.ke.com/login?service=https%3A%2F%2Fwww.ke.com%2Fuser%2Fchecklogin%3Fredirect%3Dhttps%253A%252F%252Fsh.ke.com%252Fchengjiao%252F"); + WebDriverWait wait = new WebDriverWait(driver,20) + Thread.sleep(10000) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("username")))); + + + WebElement elemUsername = driver.findElement(By.id("username")); //获取用户名框 + + WebElement elemPassword = driver.findElement(By.id("password")); //获取密码框 + + WebElement btn = driver.findElement(By.cssSelector("button[class='ant-btn btn-login ant-btn-primary']")); //获取点击登陆按钮。 + + + elemUsername.sendKeys("15021384995"); + + elemPassword.sendKeys("12qwaszx"); + + btn.submit(); // 点击行为; + } +} + + + + + + + + + + From dd68bb1364c148980b0ef66b906c403a8de0c56c Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Feb 2022 12:56:02 +0800 Subject: [PATCH 180/212] update --- .../ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +- .../script/ccbevaluation/TradeStatLog.groovy | 16 +- .../ccbevaluation/data/DistrictMarge.groovy | 83 +++++- .../ccbevaluation/data/ExcelDataTest.groovy | 54 ++-- .../script/commData/CommonDatautils.groovy | 2 +- .../script/fuzhou/FzDataResultStat.groovy | 8 +- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../fuzhou/ThirdPriceExportBatch.groovy | 7 +- .../com/fangjia/script/gj/EsHouseStat.groovy | 34 +-- .../com/fangjia/script/gj/OpenApiUtils.groovy | 28 +- .../com/fangjia/script/gj/RegionPoint.groovy | 2 +- .../gj/VagueAddressLocationBatch.groovy | 67 +++-- .../fangjia/script/gj/YouDaDataParser.groovy | 4 +- .../script/gj/data/TempMatchDistrict.groovy | 16 +- .../gj/district/FangjiaClientUtils.groovy | 2 +- .../script/gj/location/LocationTest.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 28 +- .../gj/test/CrawlDistrictIdSelect.groovy | 15 +- .../script/gj/test/FZloationUpdate.groovy | 4 +- .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 260 +++++++++--------- .../Qfang/AddMetadataBatchQfang.groovy | 10 +- .../Qfang/AddSeedBatchQfang.groovy | 2 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 13 +- .../TestCrawlTotalCount.groovy | 4 +- .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 27 files changed, 424 insertions(+), 261 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index ef1f1d1ef..2f6bdef80 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-01-1", "2022-02-1") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-02-1", "2022-03-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index 10f845a3e..f2eebcbbc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202112//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202110 - def endDate = 202112 + def sDate= 202201//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202111 + def endDate = 202201 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 1b9d19e4f..3839acde9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -39,8 +39,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 - task.exportTrade()//导出未入库成交 + task.run()//统计最近3个月抓取的成交数据量 +// task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -73,13 +73,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","10月总量","11月总量","12月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2021-10-01" - def date = "2021-11-01" - def endDate = "2021-12-01" - def newDate = "2022-01-01" + def startDate = "2021-11-01" + def date = "2021-12-01" + def endDate = "2022-01-01" + def newDate = "2022-02-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -129,7 +129,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\12月成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy index d15caa8a7..a1e095a7f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/DistrictMarge.groovy @@ -12,9 +12,90 @@ import com.fangjia.core.util.BaiDuApiUtils class DistrictMarge { static main(args) { def task = new DistrictMarge() - task.run() + //task.run() + task.repeatAddress() } + def repeatAddress(){ + def citys = CommonDatautils.findAllCity() + for(def c : citys){ + def districts = CommonDatautils.findAllDistrictByCity(c, "district", ["city", "region", "name", "lng","address","name_cat"]) + findRepead(c, districts) + } + } + def findRepead(city, districts){ + def map = [:] + for(def d : districts){ + if(d.name_cat == 0 && d.lng){ + if(d.address && d.address.contains("号")){ + def list = map.get(d.address) + if(!list){ + list = [] + map.put(d.address, list) + } + list << d + } + } + } + + def newMap = [:] + map.each {k,vals-> + if(vals.size()>=2){ + vals.sort{o1,o2-> + return o1.name.size() <=> o2.name.size() + } + newMap.put(k,vals) + } + } + + + newMap.each {k,vals-> + def names = [] + for(def d : vals){ + def newKey = d.name + if(d.name.contains("小区")){ + newKey = d.name.toString().replace("小区", "") + } + if(!names.contains(newKey)){ + names << newKey + }else{ + d.flag = "删除" + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId( d._id.toString())) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + } + } + } + + def values = newMap.values() + def allRets = [] + for(def list : values){ + allRets.addAll(list) + } + def results = [] + def delcount = 0 + for(def r : allRets){ + def rows = [] + rows << r._id.toString() + rows << r.city + rows << r.name + rows << r.address + rows << r.name_cat + if(r.flag){ + rows << r.flag + delcount++ + } + results << rows + } + // ExcelUtils.write("d:\\数据\\小区\\删除重复地址(${city}).xlsx", results, "sheet") + println "${city} 共:"+map.size()+" 2个以上:"+newMap.size()+" delcount:"+delcount + } + + def run(){ def rets = getDatas() def deleteIndex = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index 8a8a8b6c8..1df8657c9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -433,23 +433,43 @@ def city = " // } //}) -def index = 0 -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d -> - if (d['B']) { - def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"]) - if(dist){ - def alias = dist.mapping_alias - if(!alias){ - alias = [] - } - if(alias && !alias.contains(d['C'])){ - index++ - println index - alias << d['C'] - def udata = new BasicDBObject() - udata.put("mapping_alias",alias) - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) - } +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\漏映射别名.xlsx", { d -> +// if (d['B']) { +// def dist = CommonDatautils.findDistrictFieldById(d['B'],["city","region","name","mapping_alias"]) +// if(dist){ +// def alias = dist.mapping_alias +// if(!alias){ +// alias = [] +// } +// if(alias && !alias.contains(d['C'])){ +// index++ +// println index +// alias << d['C'] +// def udata = new BasicDBObject() +// udata.put("mapping_alias",alias) +// DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) +// } +// } +// } +//}) + +ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d -> + if(d['A']){ + try{ + def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"]) + if(dist){ + d['B'] = dist.city + d['C'] = dist.region + d['D'] = dist.name + + def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name) + d['E'] = price + } + }catch(Exception e){ + println d['A'] } + } + }) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 9502023d9..a25ab36f0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -218,7 +218,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","金辉伯爵山") +// q.put("name","玉泉路甲46号院") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 072e16e0a..8b5fd5715 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -23,8 +23,8 @@ class FzDataResultStat { // task.regionPriceTrendRun() } def priceMarkupRun(){ - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -2, "yyyyMM")); List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_${curMonth}.csv")) def map1 = [:] @@ -53,7 +53,7 @@ class FzDataResultStat { } rows << [keys[0], keys[1], keys[2], v.toString(), lastPrice.toString(), markup] } - //ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") + ExcelUtils.write("D:\\数据\\福州\\价格比对_${curMonth}.xlsx", rows, "数据") def regions = ["鼓楼","仓山","台江","晋安","马尾","长乐"]; @@ -94,7 +94,7 @@ class FzDataResultStat { } def prices = exportRegion(city, months) def curMonth = DateUtils.formatDateByPattern(new Date(), "yyyy-MM") - ExcelUtils.write("D:\\数据\\福州\\区域均价半年走势_${curMonth}.xlsx", prices, "数据") + ExcelUtils.write("D:\\数据\\福州\\福州6区均价半年走势_${curMonth}.xlsx", prices, "数据") } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 2f94c98c9..824cbda57 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202112.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202201.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy index 6a36f086d..7b357ab95 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdPriceExportBatch.groovy @@ -143,8 +143,8 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.zhugezhaofang_esf_price && d.zhugezhaofang_esf_price){ retMap.zhugezhaofang_esf_price = d.zhugezhaofang_esf_price } - if(!retMap.anjuke_esf_price && d.anjuke_esf_price){ - retMap.anjuke_esf_price = d.anjuke_esf_price + if(!retMap.anjuke_esf_price && d.get("58tc_esf_price")){ + retMap.anjuke_esf_price = d.get("58tc_esf_price") } if(!retMap.soufang_esf_price && d.soufang_esf_price){ retMap.soufang_esf_price = d.soufang_esf_price @@ -152,6 +152,9 @@ class ThirdPriceExportBatch extends BaseBatch{ if(!retMap.lianjia_esf_price && d.lianjia_esf_price){ retMap.lianjia_esf_price = d.lianjia_esf_price } + if(!retMap.lianjia_esf_price && d.beike_esf_price){ + retMap.lianjia_esf_price = d.beike_esf_price + } if(!retMap.cityhouse_esf_price && d.cityhouse_esf_price){ retMap.cityhouse_esf_price = d.cityhouse_esf_price diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy index b0f67784b..b78e0ec4b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy @@ -45,7 +45,7 @@ class EsHouseStat { public static void main(String[] args){ def task = new EsHouseStat(); - //task.run(); + task.run(); // task.testRun("深圳") // task.exportOpenLog(); // task.exportMongoLog() @@ -59,7 +59,7 @@ class EsHouseStat { // task.szRentHouseStat(); // task.addPrice(); - task.statNoHouseCCBData(); +// task.statNoHouseCCBData(); } def addPrice(){ @@ -337,21 +337,6 @@ class EsHouseStat { return total; } - def findHouse(def city, def startDate, def endDate){ - final StringBuffer sqlBuffer = new StringBuffer() - sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") - sqlBuffer.append(" and city = '"+city+"'") - sqlBuffer.append(" and cat = 'sell'") - sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); - ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) - def total = 0; - while (rs.next()){ - total = rs.getInt("count") - } - return total; - } - - def findHouseSource(def city, def startDate, def endDate){ final StringBuffer sqlBuffer = new StringBuffer() sqlBuffer.append("SELECT * FROM \"seawater-20*\" WHERE 1=1") @@ -373,6 +358,21 @@ class EsHouseStat { return statMap; } + + def findHouse(def city, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def total = 0; + while (rs.next()){ + total = rs.getInt("count") + } + return total; + } + def testRun(def city){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy index b5efa4f51..ba7287128 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/OpenApiUtils.groovy @@ -149,6 +149,7 @@ class OpenApiUtils { ret.lng2 = gmap.get("lng") ret.lat2 = gmap.get("lat") ret.estateType = ret.type + ret.source = "高德" }catch (Exception e){ } @@ -250,14 +251,14 @@ class OpenApiUtils { } if(type && type.contains("出入口")){ - println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}" + println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】-${type}" }else{ - println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】" + println "√高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】" poiList.add(jo); } }else{ - println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("pname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】" + println "X 高德地图:【"+jo_item.getString("cityname")+"-"+jo_item.getString("adname")+"-"+jo_item.getString("name")+"-"+jo_item.getString("address")+"】" +" 搜索:【"+city+"-"+address+"】" } } }else{ @@ -302,7 +303,7 @@ class OpenApiUtils { def distname = formatName(ret.get("name")) address = ret.get("address") if((distname && distname.contains(name)) || (distname && name.contains(distname)) || (address && address.contains(name)) || ( address && name.contains(address))){ - if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){ +// if(ret.get("area") && (region.contains(ret.get("area")) || ret.get("area").contains(region))){ lng2 = location.lng lat2 = location.lat def gmap = GeocodeUtils.geoconvBaiduToGoogle(lng2, lat2) @@ -311,7 +312,7 @@ class OpenApiUtils { def mapRegion = ret.get("area") distRegion = mapRegion districtName = distname - if(address && address.contains(mapRegion)){ + if(address && mapRegion && address.contains(mapRegion)){ address = address.substring(address.indexOf(mapRegion)+mapRegion.size()); } if(ret.get("detail_info")){ @@ -330,6 +331,7 @@ class OpenApiUtils { map.name = districtName map.region = distRegion map.estateType = estateType + map.source = "百度" rets << map lat = null @@ -340,12 +342,18 @@ class OpenApiUtils { println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】" } - }else{ - println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】" - } +// }else{ +// println "X百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+ret.get("address")+"】" +" DB:【"+city+"-"+region+"-"+name+"】" +// } }else{ - def score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name); - def score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name); + def score1 = 0 + if(distname && name){ + score1 = CosineSimilarAlgorithm.getSimilarityRatio(distname, name); + } + def score2 = 0 + if(address && name){ + score2 = CosineSimilarAlgorithm.getSimilarityRatio(address, name); + } if(score1 || score2){ rets << map println "√百度地图:【"+ret.get("city")+"-"+ret.get("area")+"-"+ret.get("name")+"-"+address+"】" +" DB:【"+city+"-"+region+"-"+name+"】(score:"+score1+"#"+score2+")" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy index efa1ead14..0ec1e9e09 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RegionPoint.groovy @@ -31,7 +31,7 @@ class RegionPointCrawl { //河北省、山西省、辽宁省、吉林省、黑龙江省、江苏省、浙江省、 // def provinces = ["安徽省","福建省","江西省","山东省","河南省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省","内蒙古自治区","广西壮族自治区","西藏自治区","宁夏回族自治区","新疆维吾尔自治区"] //def provinces = ["河北省","山西省","辽宁省","吉林省","黑龙江省","江苏省","浙江省","安徽省","福建省","江西省","山东省","河南省","湖北省","湖南省","广东省","海南省","四川省","贵州省","云南省","陕西省","甘肃省","青海省","台湾省"] - def provinces = ["内蒙古自治区","广西壮族自治区","新疆维吾尔自治区"] + def provinces = ["福建省"]//"内蒙古自治区","广西壮族自治区","新疆维吾尔自治区" for(def p : provinces){ task.crawlCityAndDistrictCode(p); task.run(p); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy index 774747a5f..b43da6c82 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy @@ -7,13 +7,17 @@ import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.mongodb.BasicDBObject +import org.bson.types.ObjectId import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet import java.util.regex.Matcher import java.util.regex.Pattern +import com.fangjia.script.util.DbUtils /** * 模糊地址经纬度更新 @@ -44,13 +48,16 @@ class VagueAddressLocationBatch extends BaseBatch { void process(Object args) { //"北京","常州", // def citys =["无锡","重庆","东莞","中山","合肥","苏州","广州","深圳","佛山","长沙","韶关","惠州","清远","珠海","福州","厦门","龙岩","漳州","成都","天津","南京","青岛"] //CommonDatautils.findAllCity(); - def citys =["上海"] //CommonDatautils.findAllCity(); - + def citys =["北京"] //CommonDatautils.findAllCity(); + //,"济南","青岛","烟台","威海","潍坊","淄博","临沂","泰安","菏泽","东营","德州","济宁","日照","聊城","滨州","枣庄" +//市谊大厦 华三新村 阳光家园 for (def city : citys) { def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name", "address", "lat2", "lat", "lng2", "lng"]); def rets = findVagueAddress(city, districts); findAndUpdateLocation(rets); - writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv") + // writeCSV(results, "d:\\数据\\结果\\" + city + "地址更新.csv") + writeCSV(results, "d:\\数据\\结果\\" + city + "地址模糊.csv") +// ExcelUtils.write("d:\\数据\\结果\\" + city + "地址模糊.xlsx", rets, "数据") } @@ -58,6 +65,7 @@ class VagueAddressLocationBatch extends BaseBatch { def findVagueAddress(def city, def districts) { def rets = [] + def rows = [] for (def district : districts) { def address = district.address; if (address) { @@ -66,10 +74,13 @@ class VagueAddressLocationBatch extends BaseBatch { //没地址 if (!address) { rets << district; + +// rows << [district.city, district.region, district.name, district.address] println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址" } else { if (!findPattern(address)) { rets << district; +// rows << [district.city, district.region, district.name, district.address] println district.city + " " + district.name + " ******[" + district.address + "]******** 模糊地址" } else { println district.city + " " + district.address + " 精确地址" @@ -97,7 +108,7 @@ class VagueAddressLocationBatch extends BaseBatch { } else if (address.indexOf("路") == -1 && address.indexOf("街") == -1 && address.indexOf("大道") == -1 && address.indexOf("巷") == -1 && address.indexOf("北里") == -1 && address.indexOf("南里") == -1 && address.indexOf("东里") == -1 && address.indexOf("西里") == -1) { return null; } else { - pattern = Pattern.compile("[路|街|大道|巷]\\d+"); + pattern = Pattern.compile("[路甲|路|街|大道|巷|弄|里]\\d+"); matcher = pattern.matcher(address) if(matcher.find()){ return s; @@ -123,16 +134,14 @@ class VagueAddressLocationBatch extends BaseBatch { for (def ret : rets) { index++; - if(ret.name!="药师新村"){ - continue - } + def locationMap = findLatAndLng(ret); if (locationMap) { updateLocation(locationMap, ret); } else { println "[没找到] " + ret.city + " " + ret.region + " " + ret.name + " " + ret.address } - println "【${index}/${rets.size()}】************************************************************************************" + println "【${index}/${rets.size()}】**************【${ret.region}---${ret.name}----${ret.address}】**********************************************************************" } } @@ -148,14 +157,14 @@ class VagueAddressLocationBatch extends BaseBatch { msgMap.db_lng2 = ret.lng2 BasicDBObject udata = new BasicDBObject() - if (ret.lat2 != locationMap.lat2) { - udata.put('lat2', locationMap.lat2) - udata.put('lng2', locationMap.lng2) + if (ret.lat2 && ret.lat2 != locationMap.lat2) { msgMap.m_lat2 = locationMap.lat2 msgMap.m_lng2 = locationMap.lng2 def distance = GeoHelper.GetDistance(ret.lat2, ret.lng2, locationMap.lat2, locationMap.lng2) msgMap.updateLngFlag = true msgMap.distance = distance + udata.put('lat2', locationMap.lat2) + udata.put('lng2', locationMap.lng2) } if (ret.lat != locationMap.lat) { @@ -171,31 +180,39 @@ class VagueAddressLocationBatch extends BaseBatch { if (udata) { println ret.city + " " + ret.region + " " + ret.name + " [ 修改其经纬度!]" udata.put("m_date", new Date()) - // DBUtils.seaweed.update(new BasicDBObject('_id',locationMap._id),new BasicDBObject('$set',udata)) + DbUtils.seaweed.update(new BasicDBObject('_id',locationMap.id),new BasicDBObject('$set',udata)) } results << msgMap } - def findStandardObject(def city, def name, def rets) { + def findStandardObject(def city, def region, def name, def rets) { def map; if (rets) { if (rets.size() > 1) { //相似里找有精确地址 for (def mapObj : rets) { - mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address); - if (mapObj.address && findPattern(mapObj.address)) { - map = mapObj; - break; + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name.contains(name) || name.contains(mapObj.name)){ + mapObj.address = com.fangjia.script.util.DataUtil.formateAddress(city, mapObj.address); + if (mapObj.address && findPattern(mapObj.address)) { + map = mapObj; + break; + } + } } } if(!map){ //交带有交叉口地址 for (def mapObj : rets) { - if (mapObj.address && (mapObj.address.contains("交汇")||mapObj.address.contains("交叉"))) { - if(!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")){ - map = mapObj; - break; + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name.contains(name) || name.contains(mapObj.name)) { + if (mapObj.address && (mapObj.address.contains("交汇") || mapObj.address.contains("交叉"))) { + if (!mapObj.address.contains("与;") && !mapObj.address.startsWith("路") && !mapObj.address.startsWith("路;")) { + map = mapObj; + break; + } + } } } } @@ -211,18 +228,18 @@ class VagueAddressLocationBatch extends BaseBatch { def findLatAndLng(ret) { //https://restapi.amap.com/v3/place/text?keywords= def rets = OpenApiUtils.getGaoDeSearchLocation(ret.city, ret.region, ret.name) - def map = findStandardObject(ret.city, ret.name, rets); + def map = findStandardObject(ret.city, ret.region, ret.name, rets); def locationMap = [:] if (!map || !map.lat) { //http://api.map.baidu.com/place/v2/search rets = OpenApiUtils.getBiDuLocation(ret.city, ret.region, ret.name); - map = findStandardObject(ret.city, ret.name, rets); + map = findStandardObject(ret.city, ret.region, ret.name, rets); } if (!map || !map.lat) { //https://map.baidu.com/?newmap=1&reqflag=pcmap&biz=1&from=webmap def code = codeMap.get(ret.city); rets = OpenApiUtils.searchBaiDuMap(ret.name, ret.region, code.toInteger()); - map = findStandardObject(ret.city, ret.name, rets); + map = findStandardObject(ret.city, ret.region, ret.name, rets); } // if((!map || !map.lat)){ @@ -237,7 +254,7 @@ class VagueAddressLocationBatch extends BaseBatch { locationMap.lng2 = (map.lng2.toDouble() * 10000000000).longValue() locationMap.id = ret._id locationMap.city = ret.city - if (ret.address.toString().endsWith("路")) { + if (map.address) { locationMap.address = map.address.replace("(","").replace(")",""); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index c43936a8a..eeb71c20d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -48,7 +48,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2021-12-01" + static def statDate = "2022-01-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2022年1月\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy index 9b1c8cfc3..13b8fdde9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/data/TempMatchDistrict.groovy @@ -12,7 +12,19 @@ class TempMatchDistrict { static main(args) { def task = new TempMatchDistrict() // task.run() - task.regionConver() +// task.regionConver() + task.stat() + } + + def stat(){ + def citys = ["广州","深圳","佛山","东莞","中山","湛江","惠州","江门","珠海","汕头","揭阳","茂名","清远","肇庆","潮州","河源","韶关","阳江","梅州","云浮","汕尾"] + def total = 0 + for(def city : citys){ + def count = CommonDatautils.findAllDistrictCount("district", city) + total = total + count + println city +" "+count + } + println "共:"+total } def regionConver(){ @@ -29,7 +41,7 @@ class TempMatchDistrict { } def run(){ def path = "D:\\数据\\小区\\收集数据-匹配\\收集数据" - def files = ["合肥.xlsx","湖北.xlsx","陕西.xlsx","上海.xlsx","深圳.xlsx","苏州.xlsx","天津.xlsx","雄安.xlsx","浙江.xlsx","重庆.xlsx"] + def files = ["杭州.xlsx"] for(def f : files){ def filePath = path+"\\"+f; match(filePath); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy index e68350d78..6b778545b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/FangjiaClientUtils.groovy @@ -10,7 +10,7 @@ class FangjiaClientUtils { public static def getMapData(def city, def region, def name, def address, def types, def mapCode){ JSONObject jsonObj = null try { - def url = "http://192.168.0.19:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode + def url = "http://192.168.0.216:38080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode //def url = "http://192.168.10.166:8080/FangJiaDataService/match?city="+city+"®ion="+region+"&name="+name+"&types="+types+"&mapCode="+mapCode if(address){ url = url + "&address="+address diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy index ce1d3ee1a..3c2061a52 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/LocationTest.groovy @@ -40,9 +40,9 @@ class LocationTest { // System.out.println(str); def task = new LocationTest() -// task.run() + task.run() // task.run2() - task.run3() +// task.run3() // task.run4() // task.run5() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index fe25a7538..1e18741df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -1,5 +1,6 @@ package com.fangjia.script.gj.paimai +import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil @@ -33,8 +34,8 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); - task.run() - // task.run1() +// task.run() +// task.run1() task.addDealRegion(); } @@ -94,12 +95,12 @@ public class LianjiaCrawlByChrome { } def run(){ -// System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); -// System.setProperty("http.nonProxyHosts", "localhost"); -// WebDriver driver = new ChromeDriver() -// def cityMap = getAllCityUrls(driver) -// getLogin(driver) -// crawlData(driver, cityMap) + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + WebDriver driver = new ChromeDriver() + def cityMap = getAllCityUrls(driver) + getLogin(driver) + crawlData(driver, cityMap) } @@ -109,7 +110,7 @@ public class LianjiaCrawlByChrome { lastMonth = lastMonth +"-01" def q = new BasicDBObject() q.put("region", null) - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2021-12-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) // q.put("source", new BasicDBObject('$in', ["链家在线","贝壳找房"])) def dbcursor = DEAL_PRICE_COLL.find(q) def keys = [] @@ -193,7 +194,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -219,6 +220,9 @@ public class LianjiaCrawlByChrome { detail.city = city detail.district_name = arrs[0] detail.region = findRegionByThirdPary(detail.city, detail.district_name) + if(detail.region){ + detail.region = RegionData.formatRegion(city, detail.region) + } def roomStr = arrs[1] if (roomStr) { @@ -391,9 +395,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(!href.contains("fang.lianjia")){ + // if(!href.contains("fang.lianjia") && e.text().contains("福州")){ cityMap.put(e.text(), href) - } + // } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 5be06fa7f..71142969a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -60,12 +60,12 @@ class CrawlDistrictIdSelect { def findCrawlData(){ def q = new BasicDBObject() - q.put("city","福州") - q.put("source", "城市房产") - q.put("s_date",202012) + q.put("city","北京") + q.put("source", "搜房") + q.put("s_date",202112) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = ".cityhouse.cn/ha/" + def prefix = "esf.fang.com/loupan/" def keyList =new HashSet() for(def db : dbcursor){ def url = null @@ -77,11 +77,14 @@ class CrawlDistrictIdSelect { url = db.url } if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".html")) + def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".htm")) if(district_id=="0"){ continue } def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix)) + if(!shortCity){ + shortCity = "bj" + } def key = shortCity +"#"+district_id def ids = retMap.get(key) @@ -92,7 +95,7 @@ class CrawlDistrictIdSelect { if(!keyList.contains(key)){ def map = new BasicDBObject() - map.root = "cityhouse.com" + map.root = "fang.com" map.city_short= shortCity map.city = db.city map.region = db.region diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy index 0f65998a6..8a490f49d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/FZloationUpdate.groovy @@ -68,7 +68,7 @@ class FZloationUpdate { Long lng0 = lng*10000000000L udata.put("lat", lat0) udata.put("lng", lng0 ) - DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) + // DbUtils.seaweed.update(new BasicDBObject("_id", dist._id), new BasicDBObject('$set', udata)) println "[修改-${total}] "+ dist.region+" "+dist.name+" "+dist.address+" "+upldateFlag } @@ -153,7 +153,7 @@ class FZloationUpdate { if(addr){ def rMap = [:] def reps = GaoDeApiUtils.geo(dist.city, addr); - if(reps && reps.count.toInteger()>=1){ + if(reps && reps.count && reps.count.toInteger()>=1){ def map = reps.geocodes.get(0) def location = map.location def level = map.level diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 5d972bd90..7f04edab1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "58同城"// - def citys = findCrawlCity(source, 202111); + def source = "城市房产"// + def citys = findCrawlCity(source, 202112); for(def city : citys){ - def count1 = findCount(source, city, 202111); - def count2 = findCount(source, city, 202112); + def count1 = findCount(source, city, 202112); + def count2 = findCount(source, city, 202201); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 46b9b6d4c..bc691499b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -39,11 +39,12 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","不","了","这","中","大","有","和","人","是","在","一","的","为","上","个","国","我","以","要","他","时","来","用","们","生","到","作","地","于","出","就","分","对","成","会","可","主","发","年","动","同","工","也","能","下","过","子","说","产","种","面","而","方","后","多","定","行","学","法","所","民","得","经","十","三","之","进","着","等","部","度","家","电","力","里","如","水","化","高","自","二","理","起","小","物","现","实","加","量","都","两","体","制","机","当","使","点","从","业","本","去","把","性","好","应","开","它","合","还","因","由","其","些","然","前","外","天","政","四","日","那","社","义","事","平","形","相","全","表","间","样","与","关","各","重","新","线","内","数","正","心","反","你","明","看","原","又","么","利","比","或","但","质","气","第","向","道","命","此","变","条","只","没","结","解","问","意","建","月","公","无","系","军","很","情","者","最","立","代","想","已","通","并","提","直","题","党","程","展","五","果","料","象","员","革","位","入","常","文","总","次","品","式","活","设","及","管","特","件","长","求","老","头","基","资","边","流","路","级","少","图","山","统","接","知","较","长","将","组","见","计","别","她","手","角","期","根","论","运","农","指","几","九","区","强","放","决","西","被","干","做","必","战","先","回","则","任","取","据","处","队","南","给","色","光","门","即","保","治","北","造","百","规","热","领","七","海","地","口","东","导","器","压","志","世","金","增","争","济","阶","油","思","术","极","交","受","联","什","认","六共","权","收","证","改","清","已","美","再","采","转","更","单","风","切","打","白","教","速","花","带","安","场","身","车","例","真","务","具","万","每","目","至","达","走","积","示","议","声","报","斗","完","类","八","离","华","名","确","才","科","张","信","马","节","话","米","整","空","元","况","今","集","温","传","土","许","步","群","广","石","记","需","段","研","界","拉","林","律","叫","且","究","观","越","织","装","影","算","低","持","音","众","书","布","复","容","儿","须","际","商","非","验","连","断","深","难","近","矿","千","周","委","素","技","备","半","办","青","省","列","习","响","约","支","般","史","感","劳","便","团","往","酸","历","市","克","何","除","消","构","府","称","太","准","精","值","号","率","族","维","划","选","标","写","存","候","毛","亲","快","效","斯","院","查","江","型","眼","王","按","格","养","易","置","派","层","片","始","却","专","状","育","厂","京","识","适","属","圆","包","火","住","调","满","县","局","照","参","红","细","引","听","该","铁","价","严","首","底","液","官","德","调","随","病","苏","失","尔","死","讲","配","女","黄","推","显","谈","罪","神","艺","呢","席","含","企","望","密","批","营","项","防","举","球","英","氧","势","告","李","台","落","木","帮","轮","破","亚","师","围","注","远","字","材","排","供","河","态","封","另","施","减","树","溶","怎","止","案","言","士","均","武","固","叶","鱼","波","视","仅","费","紧","爱","左","章","早","朝","害","续","轻","服","试","食","充","兵","源","判","护","司","足","某","练","差","致","板","田","降","黑","犯","负","击","范","继","兴","似","余","坚","曲","输","修","的","故","城","夫","够","送","笔","船","占","右","财","吃","富","春","职","觉","汉","画","功","巴","跟","虽","杂","飞","检","吸","助","升","阳","互","初","创","抗","考","投","坏","策","古","径","换","未","跑","留","钢","曾","端","责","站","简","述","钱","副","尽","帝","射","草","冲","承","独","令","限","阿","宣","环","双","请","超","微","让","控","州","良","轴","找","否","纪","益","依","优","顶","础","载","倒","房","突","坐","粉","敌","略","客","袁","冷","胜","绝","析","块","剂","测","丝","协","重","诉","念","陈","仍","罗","盐","友","洋","错","苦","夜","刑","移","频","逐","靠","混","母","短","皮","终","聚","汽","村","云","哪","既","距","卫","停","烈","央","察","烧","行","迅","境","若","印","洲","刻","括","激","孔","搞","甚","室","待","核","校","散","侵","吧","甲","游","久","菜","味","旧","模","湖","货","损","预","阻","毫","普","稳","乙","妈","植","息","扩","银","语","挥","酒","守","拿","序","纸","医","缺","雨","吗","针","刘","啊","急","唱","误","训","愿","审","附","获","茶","鲜","粮","斤","孩","脱","硫","肥","善","龙","演","父","渐","血","欢","械","掌","歌","沙","著","刚","攻","谓","盾","讨","晚","粒","乱","燃","矛","乎","杀","药","宁","鲁","贵","钟","煤","读","班","伯","香","介","迫","句","丰","培","握","兰","担","弦","蛋","沉","假","穿","执","答","乐","谁","顺","烟","缩","征","脸","喜","松","脚","困","异","免","背","星","福","买","染","井","概","慢","怕","磁","倍","祖","皇","促","静","补","评","翻","肉","践","尼","衣","宽","扬","棉","希","伤","操","垂","秋","宜","氢","套","笔","督","振","架","亮","末","宪","庆","编","牛","触","映","雷","销","诗","座","居","抓","裂","胞","呼","娘","景","威","绿","晶","厚","盟","衡","鸡","孙","延","危","胶","还","屋","乡","临","陆","顾","掉","呀","灯","岁","措","束","耐","剧","玉","赵","跳","哥","季","课","凯","胡","额","款","绍","卷","齐","伟","蒸","殖","永","宗","苗","川","炉","岩","弱","零","杨","奏","沿","露","杆","探","滑","镇","饭","浓","航","怀","赶","库","夺","伊","灵","税","了","途","灭","赛","归","召","鼓","播","盘","裁","险","康","唯","录","菌","纯","借","糖","盖","横","符","私","努","堂","域","枪","润","幅","哈","竟","熟","虫","泽","脑","壤","碳","欧","遍","侧","寨","敢","彻","虑","斜","薄","庭","都","纳","弹","饲","伸","折","麦","湿","暗","荷","瓦","塞","床","筑","恶","户","访","塔","奇","透","梁","刀","旋","迹","卡","氯","遇","份","毒","泥","退","洗","摆","灰","彩","卖","耗","夏","择","忙","铜","献","硬","予","繁","圈","雪","函","亦","抽","篇","阵","阴","丁","尺","追","堆","雄","迎","泛","爸","楼","避","谋","吨","野","猪","旗","累","偏","典","馆","索","秦","脂","潮","爷","豆","忽","托","惊","塑","遗","愈","朱","替","纤","粗","倾","尚","痛","楚","谢","奋","购","磨","君","池","旁","碎","骨","监","捕","弟","暴","割","贯","殊","释","词","亡","壁","顿","宝","午","尘","闻","揭","炮","残","冬","桥","妇","警","综","招","吴","付","浮","遭","徐","您","摇","谷","赞","箱","隔","订","男","吹","乐","园","纷","唐","败","宋","玻","巨","耕","坦","荣","闭","湾","键","凡","驻","锅","救","恩","剥","凝","碱","齿","截","炼","麻","纺","禁","废","盛","版","缓","净","睛","昌","婚","涉","筒","嘴","插","岸","朗","庄","街","藏","姑","贸","腐","奴","啦","惯","乘","伙","恢","匀","纱","扎","辩","耳","彪","臣","亿","璃","抵","脉","秀","萨","俄","网","舞","店","喷","纵","寸","汗","挂","洪","着","贺","闪","柬","爆","烯","津","稻","墙","软","勇","像","滚","厘","蒙","芳","肯","坡","柱","荡","腿","仪","旅","尾","轧","冰","贡","登","黎","削","钻","勒","逃","障","氨","郭","峰","币","港","伏","轨","亩","毕","擦","莫","刺","浪","秘","援","株","健","售","股","岛","甘","泡","睡","童","铸","汤","阀","休","汇","舍","牧","绕","炸","哲","磷","绩","朋","淡","尖","启","陷","柴","呈","徒","颜","泪","稍","忘","泵","蓝","拖","洞","授","镜","辛","壮","锋","贫","虚","弯","摩","泰","幼","廷","尊","窗","纲","弄","隶","疑","氏","宫","姐","震","瑞","怪","尤","琴","循","描","膜","违","夹","腰","缘","珠","穷","森","枝","竹","沟","催","绳","忆","邦","剩","幸","浆","栏","拥","牙","贮","礼","滤","钠","纹","弹","罢","拍","咱","喊","袖","埃","勤","罚","焦","潜","伍","墨","欲","缝","姓","刊","饱","仿","奖","铝","鬼","丽","跨","默","挖","链","扫","喝","袋","炭","污","幕","诸","弧","励","梅","奶","洁","灾","舟","鉴","苯","讼","抱","毁","率","懂","寒","智","埔","寄","届","跃","渡","挑","丹","艰","贝","碰","拔","爹","戴","码","梦","芽","熔","赤","渔","哭","敬","颗","奔","藏","铅","熟","仲","虎","稀","妹","乏","珍","申","桌","遵","允","隆","螺","仓","魏","锐","晓","氮","兼","隐","碍","赫","拨","忠","肃","缸","牵","抢","博","巧","壳","兄","杜","讯","诚","碧","祥","柯","页","巡","矩","悲","灌","龄","伦","票","寻","桂","铺","圣","恐","恰","郑","趣","抬","荒","腾","贴","柔","滴","猛","阔","辆","妻","填","撤","储","签","闹","扰","紫","砂","递","戏","吊","陶","伐","喂","疗","瓶","婆","抚","臂","摸","忍","虾","蜡","邻","胸","巩","挤","偶","弃","槽","劲","乳","邓","吉","仁","烂","砖","租","乌","舰","伴","瓜","浅","丙","暂","燥","橡","柳","迷","暖","牌","纤","秧","胆","详","簧","踏","瓷","谱","呆","宾","糊","洛","辉","愤","竞","隙","怒","粘","乃","绪","肩","籍","敏","涂","熙","皆","侦","悬","掘","享","纠","醒","狂","锁","淀","恨","牲","霸","爬","赏","逆","玩","陵","祝","秒","浙","貌","役","彼","悉","鸭","着","趋","凤","晨","畜","辈","秩","卵","署","梯","炎","滩","棋","驱","筛","峡","冒","啥","寿","译","浸","泉","帽","迟","硅","疆","贷","漏","稿","冠","嫩","胁","芯","牢","叛","蚀","奥","鸣","岭","羊","凭","串","塘","绘","酵","融","盆","锡","庙","筹","冻","辅","摄","袭","筋","拒","僚","旱","钾","鸟","漆","沈","眉","疏","添","棒","穗","硝","韩","逼","扭","侨","凉","挺","碗","栽","炒","杯","患","馏","劝","豪","辽","勃","鸿","旦","吏","拜","狗","埋","辊","掩","饮","搬","骂","辞","勾","扣","估","蒋","绒","雾","丈","朵","姆","拟","宇","辑","陕","雕","偿","蓄","崇","剪","倡","厅","咬","驶","薯","刷","斥","番","赋","奉","佛","浇","漫","曼","扇","钙","桃","扶","仔","返","俗","亏","腔","鞋","棱","覆","框","悄","叔","撞","骗","勘","旺","沸","孤","粘","吐","孟","渠","屈","疾","妙","惜","仰","狠","胀","谐","抛","霉","桑","岗","嘛","衰","盗","渗","脏","赖","涌","甜","曹","阅","肌","哩","厉","烃","纬","毅","昨","伪","症","煮","叹","钉","搭","茎","笼","酷","偷","弓","锥","恒","杰","坑","鼻","翼","纶","叙","狱","逮","罐","络","棚","抑","膨","蔬","寺","骤","穆","冶","枯","册","尸","凸","绅","坯","牺","焰","轰","欣","晋","瘦","御","锭","锦","丧","旬","锻","垄","搜","佛","扑","邀","亭","酯","迈","舒","脆","酶","闲","忧","酚","顽","羽","涨","卸","仗","陪","薄","辟","惩","杭","姚","肚","捉","飘","漂","昆","欺","吾","郎","烷","汁","呵","饰","萧","雅","邮","迁","燕","撒","姻","赴","宴","烦","削","债","帐","斑","铃","旨","醇","董","饼","雏","姿","拌","傅","腹","妥","揉","贤","拆","歪","葡","胺","丢","浩","徽","昂","垫","挡","览","贪","慰","缴","汪","慌","冯","诺","姜","谊","凶","劣","诬","耀","昏","躺","盈","骑","乔","溪","丛","卢","抹","易","闷","咨","刮","驾","缆","悟","摘","铒","掷","颇","幻","柄","惠","惨","佳","仇","腊","窝","涤","剑","瞧","堡","泼","葱","罩","霍","捞","胎","苍","滨","俩","捅","湘","砍","霞","邵","萄","疯","淮","遂","熊","粪","烘","宿","档","戈","驳","嫂","裕","徙","箭","捐","肠","撑","晒","辨","殿","莲","摊","搅","酱","屏","疫","哀","蔡","堵","沫","皱","畅","叠","阁","莱","敲","辖","钩","痕","坝","巷","饿","祸","丘","玄","溜","曰","逻","彭","尝","卿","妨","艇","吞","韦","怨","矮","歇","郊","禄","捻","漠","粹","颠","宏","冤","肪","饥","呵","仙","押","挨","醛","娃","拾","没","佩","勿","吓","讹","侯","恋","夕","锌","篡","戚","淋","蓬","岂","釉","兆","泊","魂","拘","亡","杠","摧","氟","颂","浑","凌","铀","诱","犁","谴","颁","舶","扯","嘉","萌","犹","滋","焊","舌","匹","媳","肺","掠","酿","烹","疲","驰","鸦","窄","辱","狭","朴","遣","菲","奸","韧","辣","拳","秆","卧","醉","竭","茅","墓","矣","哎","艳","敦","舆","缔","雇","尿","葬","履","契","禽","渣","衬","躲","赔","咸","溉","贼","醋","堤","抖","妃","裤","廉","晴","挽","掀","茫","丑","亥","拦","悠","阐","慧","佐","奇","竖","孝","柜","麟","绣","遥","逝","愁","肖","昭","芬","逢","窑","捷","圜","盲","闸","宙","辐","披","账","狼","幽","绸","蜂","慎","餐","酬","誓","惟","叉","弥","址","帜","芝","砌","唉","仆","涛","臭","翠","盒","劫","慨","炳","阖","寂","椒","倘","拓","畏","喉","巾","颈","垦","拚","兽","蔽","芦","乾","爽","窃","谭","挣","崩","模","褐","传","翅","儒","伞","晃","谬","胚","剖","凑","眠","浊","霜","礁","蔑","抄","闯","洒","碑","蓉","耶","猜","蹲","壶","唤","澳","锯","郡","玲","绵","纽","梳","掏","吁","锤","鼠","穴","椅","殷","遮","吵","萍","厌","畜","俱","夸","吕","囊","捧","雌","闽","饶","瞬","郁","哨","凿","朝","俺","浒","茂","肝","勋","盯","籽","耻","菊","滥","稼","戒","奈","帅","鞭","蚕","镁","询","跌","烤","坛","宅","笛","鄂","蛮","颤","棍","睁","鼎","岌","降","侍","藩","嚷","匪","岳","糟","缠","迪","泄","卑","氛","堪","萝","盛","碘","缚","悦","澄","甫","攀","屠","溢","拱","晰","携","朽","吟","菱","谦","凹","俊","芒","盼","婶","艘","酰","趁","唇","挫","羞","浴","疼","萎","肴","愚","肿","刨","绞","枢","嫁","慕","舱","铲","苹","豫","谕","迭","潘","顷","翁","榜","匠","欠","茬","畴","胃","沾","踪","弊","哼","鹏","歧","桐","沃","悼","惑","溃","蔗","荐","潭","孢","露","诊","庸","聪","嫌","厨","庞","祁","钳","肆","梭","赠","崖","篮","颖","甸","藻","捣","且","撕","诏","贞","赐","慈","炕","胖","兹","差","琼","锈","汛","卓","棵","馈","挠","灶","婴","蒂","肤","衫","沥","仑","勉","沪","逸","蜜","浦","嗓","晕","膏","祭","赢","艾","扮","鹅","怜","蒲","兔","孕","呖","蘖","挪","淑","谣","惧","廊","缅","俘","骄","膀","陡","宰","诞","峻","恼","腺","猎","涡","夷","愉","魔","铵","葛","贾","似","荫","哟","脊","钞","苛","锰","椭","镶","杏","溴","倚","滞","会","氓","捏","斩","傲","匆","僵","卤","烫","衍","榨","拢","裸","屑","咽","坊","舅","渴","翔","邪","拄","窖","猫","砌","钦","媒","脾","勺","柏","栅","噪","昼","耿","扁","辰","秤","得","贩","糕","梁","昙","衷","宦","扔","哇","诈","嘱","藤","卜","冈","悔","廓","皂","拐","氰","杉","玛","矢","寓","瓣","罕","垮","笋","淘","衔","称","恭","喇","帕","桉","秉","帘","铭","蛇","摔","斋","叭","帆","裸","俭","瘤","篷","砸","肢","辟","脖","瞪","暑","卜","竿","歼","笙","酮","蕴","哗","瞎","喀","刃","楔","喘","枚","嵌","挝","厢","粤","甩","拴","膝","恳","腕","娓","熄","锚","忌","愧","哦","荆","圃","骚","丸","蒜","毯","弗","俯","鹿","梢","屯","衙","轿","贱","垒","谅","踢","哑","滔","渥","饷","泳","棕","熬","搁","腈","梨","吻","樱","奠","捆","姨","柏","聘","惕","郓","绑","冀","裹","酥","寡","彦","稠","啡","钝","汝","擅","汰","鳙","埔","敞","嘿","逊","栋","谨","咖","鲤","雀","佣","庵","葫","贿","鳞","拼","搏","谎","塌","忉","腻","戊","怖","坟","禾","刹","嘻","桔","坎","拇","煽","狮","痒","曾","梗","寇","鹰","烛","哄","莽","雯","胳","龟","亟","糠","泌","坪","傻","什","喻","渊","蚌","跪","巷","涅","钊","譬","蕊","膛","侮","奕","枕","辫","况","扼","郝","寥","凄","厦","腥","钧","耦","蹄","戥","屁","诵","匈","桩","钓","涵","倦","袍","抒","屿","蹈","忿","敷","虹","聊","嗣","尉","灿","糙","蹬","嗯","姬","狡","笨","辜","僧","茨","讽","翰","枉","岐","枣","崭","焚","咕","猴","揽","涝","耍","趟","汹","咋","傍","镀","给","爵","虏","劈","璋","踩","瞅","迄","昔","汞","呱","诡","魄","祺","嘲","惶","赃","癌","咐","歉","扳","鄙","庐","聂","便","芡","躯","贬","煌","拧","隋","襄","淤","宠","炊","滇","謇","懒","栓","佑","憾","骆","裙","猖","兜","孵","痼","盥","曝","泣","絮","韵","眷","旷","噢","参","栖","盏","鳌","溅","煎","校","榴","暮","琪","淆","陛","巢","哒","吼","槐","唧","其","沛","乞","蜀","蜇","赚","捍","铰","幂","尧","咒","耽","叮","褂","焕","煞","雹","搓","釜","铬","拣","募","淹","瑰","鲢","茄","灼","邹","躬","觉","娇","焉","彰","鹤","琳","沦","畔","惹","庶","毙","皖","邢","禹","渍","绷","窜","翘","淫","箪","陌","膊","鞑","咳","玫","巫","拂","蕉","澜","赎","绥","锄","囱","赌","颊","缕","寅","躁","稚","庚","苟","氦","魁","珊","蜕","蛭","酌","逗","闺","蔓","撇","豌","朕","缉","襟","镍","桅","荧","侄","卒","佃","瞿","娶","饪","耸","乍","靶","痴","靖","扛","筐","韶","嚣","崔","蓿","岔","氘","娥","剿","霖","喃","搪","雍","裳","撰","豹","骏","慷"] static def totalCount = new AtomicLong() - //"福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", +// "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", // "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", // "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", // "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", -// "呼和浩特":"hu|811","济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", +// "呼和浩特":"hu|811", +// "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", // "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", // "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", // "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", @@ -53,133 +54,134 @@ class Esf58Crawl extends BaseBatch{ // "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", // "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", // "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", -// "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752", - def cityList = ["武夷山":"wuyishan|10761", - "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", - "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", - "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", - "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", - "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", - "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", - "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", - "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", - "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", - "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", - "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", - "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", - "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", - "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", - "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", - "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", - "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", - "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", - "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", - "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", - "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", - "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", - "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", - "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", - "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", - "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", - "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", - "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", - "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", - "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", - "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", - "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", - "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", - "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", - "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", - "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", - "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", - "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", - "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", - "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", - "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", - "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", - "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", - "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", - "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", - "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", - "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", - "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", - "邢台":"xt|751","衡水":"hs|993", - "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", - "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", - "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", - "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", - "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", - "沙河":"shaheshi|755", - "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", - "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", - "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", - "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", - "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", - "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", - "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", - "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", - "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", - "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", - "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", - "浮梁":"fuliangxian|9071", - "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", - "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", - "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", - "梨树县":"lishu|10176", - "鞍山":"as|523","抚顺":"fushun|5722", - "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", - "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", - "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", - "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", - "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", - "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", - "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", - "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", - "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", - "泰安":"ta|686","聊城":"lc|882", - "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", - "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", - "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", - "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", - "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", - "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", - "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", - "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", - "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", - "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", - "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", - "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", - "临朐":"linqu|374","鄄城":"juancheng|5635", - "运城":"yuncheng|5653","晋中":"jz|8854", - "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", - "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", - "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", - "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", - "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", - "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", - "德阳":"deyang|2373","宜宾":"yb|2380", - "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", - "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", - "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", - "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", - "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", - "昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", - "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", - "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", - "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", - "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", - "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", - "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", - "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", - "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", - "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", - "丽水":"lishui|7921","衢州":"quzhou|6793", - "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", - "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", - "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", - "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", - "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", - "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" +// "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", +// "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", +// "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", +// "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", +// "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", +// "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", +// "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", +// "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", +// "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", +// "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", +// "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", +// "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", +// "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", +// "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", +// "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", +// "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", +// "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", +// "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", +// "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", +// "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", +// "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", +// "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", +// "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", +// "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", +// "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", +// "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", +// "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", +// "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", +// "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", +// "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", +// "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", +// "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", +// "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", +// "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", +// "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", +// "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", +// "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", +// "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", +// "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", +// "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", +// "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", +// "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", +// "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", +// "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", +// "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", +// "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", +// "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", +// "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", +// "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", +// "邢台":"xt|751","衡水":"hs|993", +// "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", +// "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", +// "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", +// "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", +// "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", +// "沙河":"shaheshi|755", +// "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", +// "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", +// "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", +// "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", +// "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", +// "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", +// "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", +// "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", +// "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", +// "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", +// "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", +// "浮梁":"fuliangxian|9071", +// "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", +// "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", +// "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", +// "梨树县":"lishu|10176", +// "鞍山":"as|523","抚顺":"fushun|5722", +// "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", +// "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", +// "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", +// "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", +// "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", +// "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", +// "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", +// "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", +// "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", +// "泰安":"ta|686","聊城":"lc|882", +// "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", +// "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", +// "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", +// "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", +// "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", +// "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", +// "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", +// "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", +// "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", +// "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", +// "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", +// "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", +// "临朐":"linqu|374","鄄城":"juancheng|5635", +// "运城":"yuncheng|5653","晋中":"jz|8854", +// "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", +// "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", +// "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", +// "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", +// "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", +// "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", +// "德阳":"deyang|2373","宜宾":"yb|2380", +// "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", +// "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", +// "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", +// "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", +// "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", +// "昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", +// "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", +// "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", +// "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", +// "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", +// "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", +// "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", +// "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", +// "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", +// "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", +// "丽水":"lishui|7921","衢州":"quzhou|6793", +// "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", +// "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", +// "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", +// "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", +// "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", +// "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" + def cityList = [ + "重庆":"cq|37","保定":"bd|424","信阳":"xy|8694","南京":"nj|172","南宁":"nn|845","厦门":"xm|606","青岛":"qd|122","哈尔滨":"hrb|202","天津":"tj|18","太原":"ty|740","宁德":"nd|7951","宜昌":"yc|858","成都":"cd|102","惠州":"huizhou|722","扬州":"yz|637","昆明":"km|541","杭州":"hz|79","武汉":"wh|158","泰州":"taizhou|693","洛阳":"luoyang|556","深圳":"sz|4","石家庄":"sjz|241","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy index f6e5f8b79..9d1f5561f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddMetadataBatchQfang.groovy @@ -315,11 +315,11 @@ class AddMetadataBatchQfang{ private static void saveThirdPartyPrice(def saveDate){ def q = new BasicDBObject() - q.put("url", saveDate.get("xqUrl")) q.put("s_date", SHORT_DATE); q.put("city", saveDate.get("city")); q.put("region", saveDate.get("region")); q.put("name", saveDate.get("xqName")); + q.put("source", saveDate.get("source")); def udata = THIRD_PARTY_PRICE_COLL.findOne(q) if(!udata){ @@ -336,6 +336,14 @@ class AddMetadataBatchQfang{ obj.put("source", saveDate.get("source")); obj.put("c_date", new Date()); THIRD_PARTY_PRICE_COLL.save(obj); + println "[save] "+saveDate.toString() + }else{ + def updata = new BasicDBObject() + updata.put("avg_price", saveDate.get("price")) + updata.put("url", saveDate.get("xqUrl")); + updata.put("m_date", new Date()); + THIRD_PARTY_PRICE_COLL.update(q,new BasicDBObject('$set',updata)) + println "[update] "+saveDate.toString() } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy index 3584a5201..b4911d45c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Qfang/AddSeedBatchQfang.groovy @@ -87,7 +87,7 @@ class AddSeedBatchQfang extends BaseBatch{ href = "https:"+href; } if(href&&!CITYS.contains(city)){ - ret.put(city, href); + ret.put(city, href); } } }catch(Exception e){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 1aae0a14c..0b75f8545 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] //"fang.com":"搜房", - def sourceMap = ["fang.com":"搜房"] + def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() @@ -94,11 +94,16 @@ class SynchDistrictPrice { for(def r : rets){ def obj= findIsExists(r.city, r.region, r.name, SHORT_DATE, source) if(obj){ + updateIndex++ thirdPartyColl.remove(new BasicDBObject('_id',obj._id)) - addIndex++ thirdPartyColl.save(r) }else{ - thirdPartyColl.save(r) + try{ + addIndex++ + thirdPartyColl.save(r) + }catch(Exception e){ + e.printStackTrace() + } } // if(!obj){ // addIndex++ @@ -112,7 +117,7 @@ class SynchDistrictPrice { // } // } } - println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 修改:"+updateIndex + println "["+source +"-"+city + "] add:"+ addIndex+" 原抓取:"+count +" 已存在:"+updateIndex } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index a03f7ea2a..ed17686b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202111); - def count2 = findTotalCount(source, 202112); + def count1 = findTotalCount(source, 202112); + def count2 = findTotalCount(source, 202201); println source +" "+count1 +" "+count2 } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 59ad4ae49..1dec5f73e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20211201", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220101", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220101", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220201", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 From ec54ae056d692eee25f2aaa49592824fa59640ec Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Feb 2022 17:02:25 +0800 Subject: [PATCH 181/212] =?UTF-8?q?=E5=AF=BC=E6=95=B0=E6=8D=AE=E4=BF=AE?= =?UTF-8?q?=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/core/util/RandomUtil.java | 2 +- .../ccbevaluation/BigdataDealBatch.groovy | 6 +- .../ccbevaluation/DistrictDelete.groovy | 180 +++++++++++++++++- .../DistrictGuidePriceBatch.groovy | 2 +- .../ccbevaluation/ExportOracleData.groovy | 35 ++-- .../script/ccbevaluation/LoadGuide.groovy | 13 +- .../fangjia/script/ccbevaluation/Test.groovy | 57 +++--- .../gj/paimai/LianjiaCrawlByChrome.groovy | 26 ++- 8 files changed, 263 insertions(+), 58 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java index 25f4d5a12..234c29d79 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/RandomUtil.java @@ -39,7 +39,7 @@ public final class RandomUtil extends RandomUtils { public static void main(String[] args) { for (int i = 0; i < 1000; i++) { - System.out.println(random(0, 10)); + System.out.println(random(-9, 9)); } } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index e38ff62f4..9cb9003d2 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -54,9 +54,9 @@ class BigdataDealBatch extends BaseBatch{ //def weekend = DateUtils.parseFromTightByPattern("2021-07-25","yyyy-MM-dd")//DateUtils.getFirstDayOfWeek(new Date()) def citys = CcbCommonUtils.citys; - for(def city : citys){ - addDealPriceMatch(city); - } +// for(def city : citys){ +// addDealPriceMatch(city); +// } println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy index 1dbc5cd4d..eaef4e568 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictDelete.groovy @@ -1,15 +1,20 @@ package com.fangjia.script.ccbevaluation +import com.fangjia.core.RegionData +import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject +import com.mongodb.DBCollection import com.mongodb.DBObject import groovy.sql.Sql import org.apache.lucene.queryparser.classic.QueryParser +import org.apache.poi.ss.util.RegionUtil import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder @@ -37,15 +42,74 @@ class DistrictDelete { dataSource.setProperties(connectionProperties) } - def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.100:31521:orcl", + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + static main(args) { def task = new DistrictDelete() // task.run() // task.gzRun(); // task.deleteDistrict() - task.deleteCommPrice() +// task.deleteCommPrice() +// task.distHouseCountStat() + task.addThirdPartyStat() +// task.processRegion() + } + def processRegion(){ + ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\重庆3个月房源.xlsx",{d-> + if(d['B']){ + def region = RegionData.formatRegion("重庆",d['B']) + if(region){ + d['B'] = region + } + } + },0) + } + + def addThirdPartyStat(){ + def datas= ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\上海12月房源.xlsx",0,1) + def idMaps = [:] + for(def d : datas){ + def id = d['A'] + idMaps.put(id, d['D']) + } + def dists = CcbCommonUtils.findAllDistrictByCity("上海", "district", ["city","region", "name", "alias","address","lng","build_type","finish_building_date","estate_type2","name_cat"]) + def totalCount = new AtomicLong() + def map = findThirdPartyPrice("上海") + final CountDownLatch latch = new CountDownLatch(dists.size()) + def thirdLists = Collections.synchronizedList(new ArrayList()); + thirdLists << ["ID","区域","名称","总量"] + for(def i=0;i3000){ + prices << db.zhugezhaofang_esf_price + } + if(db.anjuke_esf_price && db.anjuke_esf_price>3000){ + prices << db.anjuke_esf_price + } + if(db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + prices << db.get("58tc_esf_price") + } + if(db.soufang_esf_price && db.soufang_esf_price>3000){ + prices << db.soufang_esf_price + } + if(db.lianjia_esf_price && db.lianjia_esf_price>3000){ + prices << db.lianjia_esf_price + } + if(db.cityhouse_esf_price && db.cityhouse_esf_price>3000){ + prices << db.cityhouse_esf_price + } + + if(db.maitian_esf_price && db.maitian_esf_price>3000){ + prices << db.maitian_esf_price + } + + if(db.beike_esf_price && db.beike_esf_price>3000){ + prices << db.beike_esf_price + } + + if(db.get("365house_esf_price") && db.get("365house_esf_price")>3000){ + prices << db.get("365house_esf_price") + } + + if(db.zhongyuan_esf_price && db.zhongyuan_esf_price>3000){ + prices << db.zhongyuan_esf_price + } + + if(db.tuboshi_esf_price && db.tuboshi_esf_price>3000){ + prices << db.tuboshi_esf_price + } + + if( db.qfang_esf_price && db.qfang_esf_price>3000){ + prices << db.qfang_esf_price + } + + if(db.fangtan_esf_price && db.fangtan_esf_price>3000){ + prices << db.fangtan_esf_price + } + + if(db.fangduoduo_esf_price && db.fangduoduo_esf_price>3000){ + prices << db.fangduoduo_esf_price + } + + if(db.get("5i5j_esf_price") && db.get("5i5j_esf_price")>3000){ + prices << db.get("5i5j_esf_price") + } +// if(db.zhugezhaofang_esf_price || db.anjuke_esf_price ||db.soufang_esf_price || db.lianjia_esf_price || db.cityhouse_esf_price || db.maitian_esf_price || db.beike_esf_price || db.get("365house_esf_price") || db.zhongyuan_esf_price || db.tuboshi_esf_price || db.qfang_esf_price || db.fangtan_esf_price || db.fangduoduo_esf_price || db.get("5i5j_esf_price")){ +// map.put(key, db) +// } + } + + return map + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index 2f73eda31..f67de27ae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findGuidePrice(city, "2022-01-1", "2022-02-1") + def prices = CcbCommonUtils.findGuidePrice(city, "2022-02-1", "2022-03-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 1bd63cbeb..1691b8656 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -22,25 +22,24 @@ class ExportOracleData { def run() { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); - def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); // //不会变 -// exportEvaluationRateData() -// exportBigdataPoiData() -// exportBigdataDistrictData() + exportEvaluationRateData() + exportBigdataPoiData() + exportBigdataDistrictData() // -// exportGuidedPrice()//随人工改变 -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 -// -// //处理小区情况 -// exportBigdataCommunity() -// exportCommunityStaging() -// exportCommunityBuileType() + exportGuidedPrice()//随人工改变 + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 -// exportThirdPartyPrice2(lastMonth) -// exportBigdataForsale("2021-12-19") -// exportBigdataDeal("2021-12-01") +// //处理小区情况 + exportBigdataCommunity() + exportCommunityStaging() + exportCommunityBuileType() + + exportThirdPartyPrice2(lastMonth) + exportBigdataForsale("2022-01-19") + exportBigdataDeal("2022-01-01") //计算结果相关 @@ -56,11 +55,11 @@ class ExportOracleData { // exportCommunityPdescribeGj(curMonth) // exportCommCaseDetailGj(curMonth) - def curDate = curMonth.toString()+"28" - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// def curDate = curMonth.toString()+"28" +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); //ok文件 - exportLastFile() +// exportLastFile() } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy index 84f2232df..42ad78607 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/LoadGuide.groovy @@ -23,9 +23,10 @@ class LoadGuide { // saveChengduoPrice("D:\\数据\\建行\\房产估值\\指导价\\成都部分住宅小区二手房成交参考价格201-286-258.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\绍兴指导价小区_0909.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\宁波交易参考价格.xlsx") - savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") +// savePrice("D:\\数据\\建行\\房产估值\\指导价\\西安指导价2-103.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\无锡100 广州96 二手住房成交参考价格表.xlsx") // savePrice("D:\\数据\\建行\\房产估值\\指导价\\深圳指导价 - 210909 -.xlsx") + savePrice("D:\\数据\\建行\\房产估值\\指导价\\三亚指导价30.xlsx") } def match(def filename){ @@ -48,15 +49,15 @@ class LoadGuide { def datas = ExcelUtils.read(filePath,0, 1) def rets = [] for(def d : datas){ - if(d['E']){ + if(d['D']){ def o = new BasicDBObject() - o.city = d['E'] + o.city = d['D'] o.region = d['A'] o.name = d['B'] o.avg_price = d['C'].toString().toDouble().toInteger() - o.match_city = d['E'] - o.match_region = d['F'] - o.match_name = d['G'] + o.match_city = d['D'] + o.match_region = d['E'] + o.match_name = d['F'] o.source_type = 1 //1:成交参考价 2:参考价 o.c_date = new Date() rets << o diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 53a43fb77..dc639ef71 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01") -def testMap = findCaluateCityPPE(sqlInstance236, 202201) -ExcelUtils.modify(fileName,{d-> - if(d['C']){ - def oldcount = dealMap.get(d['C']) - d['T'] = oldcount - def casecount = testMap.get(d['C']) - d['U'] = casecount - println d - } -},0) +//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +//def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01") +//def testMap = findCaluateCityPPE(sqlInstance236, 202201) +//ExcelUtils.modify(fileName,{d-> +// if(d['C']){ +// def oldcount = dealMap.get(d['C']) +// d['T'] = oldcount +// def casecount = testMap.get(d['C']) +// d['U'] = casecount +// println d +// } +//},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -270,20 +270,25 @@ ExcelUtils.modify(fileName,{d-> // } //},0) -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\标注为11的数据(5034)_0118.xlsx",{d-> -// if(d['A']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['A'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -//},0) +def index = 0 +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> + def flag = d['M'] + if(flag==0.0){ + if(d['D']){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['D'])) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + index++ + println index + } + } + +},0) + //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ // def region = d['B'] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 1e18741df..c6eec9d7e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -36,7 +36,8 @@ public class LianjiaCrawlByChrome { def task = new LianjiaCrawlByChrome(); // task.run() // task.run1() - task.addDealRegion(); +// task.addDealRegion(); + task.standardRegion() } def run1(){ @@ -105,6 +106,28 @@ public class LianjiaCrawlByChrome { } + def standardRegion(){ + + def q = new BasicDBObject() + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) + def dbcursor = DEAL_PRICE_COLL.find(q) + for(def d : dbcursor){ + if(d.region){ + def newRegion = RegionData.formatRegion(d.city, d.region) + if(newRegion && newRegion!=d.region){ + def query = new BasicDBObject() + query.put("city", d.city) + query.put("region", d.region) + def data = new BasicDBObject() + data.put("region", newRegion) + DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) + } + } + + } + } + def addDealRegion(){ def lastMonth = DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyy-MM"), -1, "yyyy-MM") lastMonth = lastMonth +"-01" @@ -132,6 +155,7 @@ public class LianjiaCrawlByChrome { def data = new BasicDBObject() data.put("region", dist.region) + println m.city+ " "+m.name +" "+dist.region def ds = DEAL_PRICE_COLL.count(query) DEAL_PRICE_COLL.update(query, new BasicDBObject('$set',data), false, true) } From e27f23a299148b76527dd95470604df99730ef59 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Thu, 24 Feb 2022 17:25:10 +0800 Subject: [PATCH 182/212] =?UTF-8?q?=E4=BC=B0=E5=80=BC=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E5=88=862=E4=B8=AA=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataDealBatch.groovy | 2 +- .../ccbevaluation/CcbCommonUtils.groovy | 8 +- .../ccbevaluation/ExportOracleData.groovy | 6 +- .../ExportOracleEvaluationData.groovy | 415 ++++++++++++++++++ .../fangjia/script/ccbevaluation/Test.groovy | 60 +-- .../script/ccbevaluation/TradeStatLog.groovy | 30 +- .../ccbevaluation/data/MapAddressMatch.groovy | 89 ++++ .../script/gj/district/ZsNoExistsToDB.groovy | 2 +- .../gj/location/DistrictLocationUpdate.groovy | 176 +++++++- 9 files changed, 738 insertions(+), 50 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MapAddressMatch.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy index 9cb9003d2..aa12539b9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataDealBatch.groovy @@ -57,7 +57,7 @@ class BigdataDealBatch extends BaseBatch{ // for(def city : citys){ // addDealPriceMatch(city); // } - println citys.unique().size() +// println citys.unique().size() def rows = [] def headers = []//["城市ID","成交ID","城市名称","行政区ID","行政区名称","街道","门牌号","小区ID","小区名称","楼栋ID","楼栋编号","楼栋名称","单元ID","单元编号","单元名称","户ID","详细地址","建筑年代","权属","权属年限","期数","所在楼层","层类型","总楼层数","建筑类型","建筑结构","建筑面积","套内面积","户型","室","厅","厨","卫","房屋用途","室内结构","朝向","装修情况","评估编号","成交单价","成交总价","申报总价","评估总价","网签合同编号","交易日期","交易类型","出售付款方式","成交周期","是否有电梯","赠送面积(地面花园)","赠送面积(露台)","赠送面积(阳台)","赠送面积(飘窗)","赠送面积(阁楼)","赠送面积(车库)","赠送面积(地下室)","赠送面积(其他)","房源数据来源","中介公司ID","中介公司名称","备注","创建时间","更新时间","来源公司","成交金额","数据时间"] def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 42f56e208..78153a5f0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -149,8 +149,8 @@ class CcbCommonUtils { public static def findTradesDetailById(dist, startDate, endDate){ def q = new BasicDBObject() - q.put("city", dist.city) - q.put("region", dist.region) + // q.put("city", dist.city) + // q.put("region", dist.region) q.put("area", new BasicDBObject('$gte', 20)) q.put("avg_price", new BasicDBObject('$gt', 0)) q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) @@ -161,7 +161,7 @@ class CcbCommonUtils { def lists = [] println dist.name +" "+total for(def d : dbcursor){ - def key=d.city+"##"+d.region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") if(!rets.contains(key)){ rets << key d.dist_id = dist._id.toString() @@ -367,7 +367,7 @@ class CcbCommonUtils { q.put("visible",0) q.put("cat", cat); q.put("city", city); -// q.put("name","长隆小区") +// q.put("name","特房锦绣一方") // q.put("name_cat", 0); // q.put("estate_type2", new BasicDBObject('$in', ["普通住宅","商住"])) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 1691b8656..20e49d87c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -21,14 +21,14 @@ class ExportOracleData { } def run() { - def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); +// def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); -// //不会变 + //不会变 exportEvaluationRateData() exportBigdataPoiData() exportBigdataDistrictData() -// + exportGuidedPrice()//随人工改变 exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy new file mode 100644 index 000000000..603a00f36 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -0,0 +1,415 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql +import com.fangjia.script.ccbevaluation.ronghe.SSHHelper + +class ExportOracleEvaluationData { + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new ExportOracleEvaluationData() + task.run() + } + + def run() { + + def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); + + //计算结果相关 + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) + + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) + + exportCommunityPdescribeGj(curMonth) + exportCommCaseDetailGj(curMonth) + + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); + //ok文件 + exportOkFile() + } + + def exportAvmEvaluationGJNB(yearMonth){ + def sql = "select ID,CITYID,CITYNAME, YEAR_MONTH, COMMUNITY_NUM, HAS_PRICE_COMMUNITY_NUM, PRICE_COVERAGE, ARTIFICIAL_CALC_NUM, ARTIFICIAL_CALC_COVERAGE, SAMPLE_CALC_NUM, SAMPLE_CALC_COVERAGE, \n" + + "ANN_CALC_NUM, ANN_CALC_COVERAGE, TEST_NUM, TEST_COMM_NUM, TEST_COMM_COVERAGE, PPE15_COVERAGE, RATE_MEDIAN, COEFFICIENT,DEAL_TEST_CASE_NUM, DEAL_TEST_COMM_NUM, DEAL_TEST_COMM_COVERAGE, DEAL_PPE15_COVERAGE, \n" + + "DEAL_COEFFICIENT, DEAL_RATE_MEDIAN, THIRD_TEST_CASE_NUM, THIRD_TEST_COMM_NUM, THIRD_TEST_COMM_COVERAGE, THIRD_PPE15_COVERAGE, THIRD_COEFFICIENT, THIRD_RATE_MEDIAN, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from avm_evaluation_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.COMMUNITY_NUM + data << r.HAS_PRICE_COMMUNITY_NUM + data << r.PRICE_COVERAGE + data << r.ARTIFICIAL_CALC_NUM + data << r.ARTIFICIAL_CALC_COVERAGE + data << r.SAMPLE_CALC_NUM + data << r.SAMPLE_CALC_COVERAGE + data << r.ANN_CALC_NUM + data << r.ANN_CALC_COVERAGE + data << r.TEST_NUM + data << r.TEST_COMM_NUM + data << r.TEST_COMM_COVERAGE + data << r.PPE15_COVERAGE + data << r.RATE_MEDIAN + data << r.COEFFICIENT + data << r.DEAL_TEST_CASE_NUM + data << r.DEAL_TEST_COMM_NUM + data << r.DEAL_TEST_COMM_COVERAGE + data << r.DEAL_PPE15_COVERAGE + data << r.DEAL_COEFFICIENT + data << r.DEAL_RATE_MEDIAN + data << r.THIRD_TEST_CASE_NUM + data << r.THIRD_TEST_COMM_NUM + data << r.THIRD_TEST_COMM_COVERAGE + data << r.THIRD_PPE15_COVERAGE + data << r.THIRD_COEFFICIENT + data << r.THIRD_RATE_MEDIAN + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","YEAR_MONTH","COMMUNITY_NUM","HAS_PRICE_COMMUNITY_NUM","PRICE_COVERAGE","ARTIFICIAL_CALC_NUM","ARTIFICIAL_CALC_COVERAGE","SAMPLE_CALC_NUM","SAMPLE_CALC_COVERAGE", + //"ANN_CALC_NUM","ANN_CALC_COVERAGE","TEST_NUM","TEST_COMM_NUM","TEST_COMM_COVERAGE","PPE15_COVERAGE","RATE_MEDIAN","COEFFICIENT","DEAL_TEST_CASE_NUM","DEAL_TEST_COMM_NUM","DEAL_TEST_COMM_COVERAGE","DEAL_PPE15_COVERAGE", + //"DEAL_COEFFICIENT","DEAL_RATE_MEDIAN","THIRD_TEST_CASE_NUM","THIRD_TEST_COMM_NUM","THIRD_TEST_COMM_COVERAGE","THIRD_PPE15_COVERAGE","THIRD_COEFFICIENT","THIRD_RATE_MEDIAN","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_GJNB 上传Ftp完成......" + } + + def exportAVMEvaluationItemGJNB(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, CONFIDENCE,TARGET_AVG_PRICE, \n" + + "to_char(DEALDATE,'yyyy-mm-dd hh24:mi:ss') as DEALDATE, UNIT_PRICE, BUILDAREA, FITMENT, HOUSEASPECT, FLOORNUM, FLOORCNT, DATA_TYPE, RATE, PE, \n" + + "to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME from avm_evaluation_item_gj where YEAR_MONTH = ${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.CONFIDENCE + data << r.TARGET_AVG_PRICE + data << r.DEALDATE + data << r.UNIT_PRICE + data << r.BUILDAREA + data << r.FITMENT + data << r.HOUSEASPECT + data << r.FLOORNUM + data << r.FLOORCNT + data << r.DATA_TYPE + data << r.RATE + data << r.PE + data << r.CREATETIME + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_AVM_EVALUATION_ITEM_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","CONFIDENCE","TARGET_AVG_PRICE","DEALDATE","UNIT_PRICE","BUILDAREA","FITMENT","HOUSEASPECT","FLOORNUM","FLOORCNT","DATA_TYPE","RATE","PE","CREATETIME"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "AVM_EVALUATION_ITEM_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) + println "AVM_EVALUATION_ITEM_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePrice(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + def fileName1 = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJNB_ADD_${curDate}_0001.dat" + fileName1 = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + FileUtils.batchCreateFile(allRows, fileName1, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJNB 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + CcbFtpUtils.ftpUpload(null, fileName1, curDate) + println "COMMUNITY_BASE_PRICE_GJNB 上传Ftp完成......" + } + + def exportCommunityBasePriceGj(yearMonth){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, COMMID, COMMNAME, YEAR_MONTH, CALC_AVG_PRICE, AVG_PRICE, CALC_TYPE, FAKE_RATE, DIFF_RATE, CONFI_RATE, CORR_RATE, CONFIDENCE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from community_base_price_gj where year_month=${yearMonth}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.COMMID + data << r.COMMNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.AVG_PRICE + data << r.CALC_TYPE + data << r.FAKE_RATE + data << r.DIFF_RATE + data << r.CONFI_RATE + data << r.CORR_RATE + data << r.CONFIDENCE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28"; + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_BASE_PRICE_GJ_ADD_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "COMMUNITY_BASE_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "COMMUNITY_BASE_PRICE_GJ 上传Ftp完成......" + } + + def exportCityPrice(month){ + def sql = "select ID, CITYID, CITYNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from city_price_gj where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_CITY_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "CITY_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "CITY_PRICE_GJ 上传Ftp完成......" + } + + def exportRegionPrice(month){ + def sql = "select ID, CITYID, CITYNAME, DISTID, DISTNAME, YEAR_MONTH, CALC_AVG_PRICE, PRICE, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME, to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME, SOURCECODE from DISTRICT_PRICE_GJ where year_month=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.YEAR_MONTH + data << r.CALC_AVG_PRICE + data << r.PRICE + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_DISTRICT_PRICE_GJ_ALL_${curDate}_0001.dat" + } + def headers = []//["ID","CITYID","CITYNAME","DISTID","DISTNAME","COMMID","COMMNAME","YEAR_MONTH","CALC_AVG_PRICE","AVG_PRICE","CALC_TYPE","FAKE_RATE","DIFF_RATE","CONFI_RATE","CORR_RATE","CONFIDENCE","CREATETIME","UPDATETIME","SOURCECODE"] + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); + println "DISTRICT_PRICE_GJ 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "DISTRICT_PRICE_GJ 上传Ftp完成......" + } + + def exportCommunityPdescribeGj(month){ + +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMMUNITY_P_DESCRIBE_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] +// +// for(def city : citys) { +// def cityCode = cityCodeDataMap.get(city) + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; + def allRows = [] + sqlInstance.eachRow(sql, { r -> + def data = [] + data << r.ID + data << r.COMMUNITY_ID + data << r.CITYID + data << r.CITY_NAME + data << r.DISTRICT_CODE + data << r.DISTRICT_NAME + data << r.COMMUNITY_NAME + data << r.PQ_NAME + data << r.BASE_PRICE + data << r.CONFIDENCE_LEVEL + data << r.CNT + data << r.MIN_PRICE + data << r.MEDIANPRICE + data << r.MEANPRICE + data << r.MAX_PRICE + data << r.TIME + data << r.FIRSTDATAT + data << r.THIRDDATA + data << r.RELATIVEERROR + data << r.SKEWNESS + data << r.PQ_PRICE_RANGE + data << r.DISTRICT_PRICE_RANGE + data << r.DESCRIPTION + data << r.PRIV_BASE_PRICE + data << r.IS_ARTTFICIAL + data << r.PRIV_FINAL_PRICE + data << r.CREATE_DATE + data << r.PRICE + data << r.PRICE_SOURCE + data << r.CASE_MD_PRICE + data << r.CASE_AVG_PRICE + data << r.INDEX_LABEL + data << r.MAIN_PROPERTY + data << r.COMMTYPE + data << r.NEW_DEAL_PRICE + data << r.NEW_DEAL_DATE + data << r.PRICE_MOM_RATIO + data << r.MAX_MOM_RATIO + data << r.PRICE_YOY_RATIO + data << r.SATUR_RATIO + data << r.NEIGHBOR_PRICE + allRows << data + }); + println allRows.size() + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", false); +// } + println "community_p_describe_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "community_p_describe_gj 上传Ftp完成......" + } + + def exportCommCaseDetailGj(month){ + + def citys = CcbCommonUtils.citys; + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def fileName = "/ccb/MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\MODEL_000000_000000_COMM_CASE_DETAIL_GJ_ADD_${curDate}_0001.dat" + } + def headers = [] + + for(def city : citys){ + def cityCode = cityCodeDataMap.get(city) + def sql = "select ID, CITYID, COMMUNITY_ID, NETLISTING_ID, PRICE_DATE, to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE from comm_case_detail_gj where cityid='${cityCode}' and price_date=${month}"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.CITYID + data << r.COMMUNITY_ID + data << r.NETLISTING_ID + data << r.PRICE_DATE + data << r.CREATE_DATE + allRows << data + if(allRows.size()%200000==0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + allRows.clear() + } + }); + if(allRows.size()>0){ + FileUtils.batchCreateFile(allRows, fileName, headers, "utf-8", true); + } + } + + println "comm_case_detail_gj 数据文件输出完成......" + CcbFtpUtils.ftpUpload(null, fileName, curDate) + println "comm_case_detail_gj 上传Ftp完成......" + } + + def exportOkFile(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"28" + def dataCodeFileName = "/ccb/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpUtils.ftpUpload(null, dataCodeFileName, curDate) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index dc639ef71..146c1f07b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -//def dealMap = findDealCount(sqlInstance236, "2021-12-01", "2022-01-01") -//def testMap = findCaluateCityPPE(sqlInstance236, 202201) -//ExcelUtils.modify(fileName,{d-> -// if(d['C']){ -// def oldcount = dealMap.get(d['C']) -// d['T'] = oldcount -// def casecount = testMap.get(d['C']) -// d['U'] = casecount -// println d -// } -//},0) +def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") +def testMap = findCaluateCityPPE(sqlInstance236, 202202) +ExcelUtils.modify(fileName,{d-> + if(d['C']){ + def oldcount = dealMap.get(d['C']) + d['V'] = oldcount + def casecount = testMap.get(d['C']) + d['W'] = casecount + println d + } +},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -270,24 +270,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) -def index = 0 -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> - def flag = d['M'] - if(flag==0.0){ - if(d['D']){ - def udata = new BasicDBObject() - udata.put("name_cat", 11) - udata.put("m_date",new Date()) - - def q = new BasicDBObject() - q.put("_id", new ObjectId(d['D'])) - DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) - index++ - println index - } - } - -},0) +//def index = 0 +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> +// def flag = d['M'] +// if(flag==0.0){ +// if(d['D']){ +// def udata = new BasicDBObject() +// udata.put("name_cat", 11) +// udata.put("m_date",new Date()) +// +// def q = new BasicDBObject() +// q.put("_id", new ObjectId(d['D'])) +// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) +// index++ +// println index +// } +// } +// +//},0) //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 3839acde9..621eac1ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -9,6 +9,7 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.QueryOperators import groovy.sql.Sql import org.bson.types.ObjectId @@ -39,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 +// task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -48,6 +49,7 @@ class TradeStatLog { // task.deleteDealData() // task.deleteDealData2() // task.statTableCount() + task.filterTrade() } @@ -479,8 +481,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2021-12-01" - def endDate = "2022-01-01" + def startDate = "2022-01-01" + def endDate = "2022-02-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) @@ -561,7 +563,7 @@ class TradeStatLog { } println city +" "+rets.size() } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(12月).xlsx", rets,"数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx", rets,"数据") } def deleteDistrict(){ @@ -706,4 +708,24 @@ class TradeStatLog { } + def filterTrade(){ + ExcelUtils.modify("D:\\数据\\建行\\房产估值\\成交\\成交未入库记录(1月).xlsx",{d-> + if(!d['J']){ + def count = findTradeById(d['A']) + if(count){ + d['J'] = "其它原因过滤" + } + } + },0) + } + + + def findTradeById(id){ + def q = new BasicDBObject() + q.put("_id", new ObjectId(id)) + q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, true)) + def count = DEAL_PRICE_COLL.count(q) + return count + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MapAddressMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MapAddressMatch.groovy new file mode 100644 index 000000000..00efe7807 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MapAddressMatch.groovy @@ -0,0 +1,89 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils + +class MapAddressMatch { + static main(args) { + def task = new MapAddressMatch() + task.run() + } + + def run(){ + def city = "北京" + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\北京地址重复数据.xlsx", { d -> + index++ + println index +" "+ d['D']+" "+d['E'] + if(!d['G']){ +// def gdData = FangjiaClientUtils.getMapData(city, d['D'],d['E'], d['K'],"", "高德") +// if(gdData){ +// d['F'] = gdData.district +// d['G'] = gdData.name +// d['H'] = gdData.address +// d['I'] = gdData.type +// d['J'] = "高德" +// }else{ + def rets = OpenApiUtils.getGaoDeSearchLocation(city,d['D'],d['E']) + def map = findStandardObject(city,d['D'],d['E'], rets); + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(city,d['D'],d['E']); + map = findStandardObject(city,d['D'],d['E'], rets); + } + if(map){ + if(map.adname){ + d['F'] = map.adname + }else if(map.region){ + d['F'] = map.region + } + d['G'] = map.name + d['H'] = map.address + if(map.type){ + d['I'] = map.type + }else if(map.estateType){ + d['I'] = map.estateType + } + + d['J'] = map.source + } +// } + } + + },0) + } + + + + def findStandardObject(def city, def region, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + for (def mapObj : rets) { + if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){ + continue; + } +// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){ + map = mapObj; + break; + }else if(mapObj.address && mapObj.address.contains(name)){ + map = mapObj; + break; + }else if(mapObj.name && name){ + double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name); + if(score1){ + map = mapObj; + break; + } + } +// } + } + } + } + return map; + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy index ce6949835..5c87fa8cb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ZsNoExistsToDB.groovy @@ -134,7 +134,7 @@ class ZsNoExistsToDB { def run(){ def rets = findOpenLog2() - ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据") + // ExcelUtils.write("D:\\数据\\浙商\\不存在小区2.xlsx", rets, "数据") println rets.size() // matchMap2() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy index 0ad5dbfcf..90712ca06 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy @@ -1,11 +1,16 @@ package com.fangjia.script.gj.location +import com.fangjia.core.util.GeoUtils import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.district.NameProcess +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.ExcelUtils import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet - +import com.fangjia.script.gj.district.FangjiaClientUtils class DistrictLocationUpdate { public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000" @@ -19,20 +24,177 @@ class DistrictLocationUpdate { static main(args) { def task = new DistrictLocationUpdate() - task.run() +// task.run() + task.mapMatch() } def run(){ - def citys = CommonDatautils.findAllCity() + def citys = ["福州"]//CommonDatautils.findAllCity() for(def city : citys){ - def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","alias", "lng", "lat"]) - - - updateLocation(city, dists); + def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region","name","address","alias", "lng", "lat"]) + def rets = [] + rets << ["城市","区域","名称","地址","经度","纬度","高德区域","高德名称","高德地址","高德类型","高德经度","高德纬度","高德类型","得分","距离"] + for(def d : dists){ + def lng = null + def lat = null + if(d.lng){ + lng = d.lng/10000000000.0 + lat = d.lat/10000000000.0 + } + rets << [d.city, d.region, d.name, d.address, lng,lat] + } + ExcelUtils.write("D:\\数据\\小区\\经纬度\\${city}.xlsx",rets, "数据") } } + def mapMatch(){ + def index = 0 + ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d-> + if(!d['H'] && d['C']=="凤凰小区"){ + index++ +// def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德") +// println d['A']+" "+d['B']+" "+d['C'] +// if(gdData){ +// d['G'] = gdData.district +// d['H'] = gdData.name +// d['I'] = gdData.address +// d['J'] = gdData.type +// d['K'] = gdData.lng +// d['L'] = gdData.lat +// d['M'] = gdData.score +// if(d['E'] && d['K']){ +// def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble()) +// d['N'] = distance +// } +// } + + def rets = OpenApiUtils.getGaoDeSearchLocation(d['A'],d['B'],d['C']) + def map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(d['A'],d['B'],d['C']); + map = findStandardObject2(d['A'],d['B'],d['C'], d['D'], rets); + } + if(map){ + if(map.adname){ + d['G'] = map.adname + }else if(map.region){ + d['G'] = map.region + } + d['H'] = map.name + d['I'] = map.address + if(map.type){ + d['J'] = map.type + }else if(map.estateType){ + d['J'] = map.estateType + } + d['K'] = map.lng + d['L'] = map.lat + d['M'] = map.score + if(d['E'] && d['K']){ + def distance = GeoUtils.calculateDistance(d['E'].toDouble(), d['F'].toDouble(), d['K'].toDouble(), d['L'].toDouble()) + d['N'] = distance + } + } + } + },0) + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + score1 = Eval.me(score1.toString()) + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } + + def findStandardObject(def city, def region, def name, def rets) { + def map; + if (rets) { + if (rets.size() >= 1) { + for (def mapObj : rets) { + if(mapObj.name && (mapObj.name.contains("(公交站)")||mapObj.name.contains("门)")||mapObj.name.contains("停车场"))){ + continue; + } +// if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + if(mapObj.name && (mapObj.name.contains(name) || name.contains(mapObj.name))){ + map = mapObj; + break; + }else if(mapObj.address && mapObj.address.contains(name)){ + map = mapObj; + break; + }else if(mapObj.name && name){ + double score1 = CosineSimilarAlgorithm.getSimilarity(mapObj.name,name); + if(score1){ + map = mapObj; + break; + } + } +// } + } + } + } + return map; + } + + def queryPoiMap(def city){ final StringBuffer sqlBuffer = new StringBuffer() sqlBuffer.append("SELECT cityname,adname,name,address,type,location,m_date,typecode FROM amap_poi WHERE 1=1") From 675783d31dfacc0f94cb3dc2cb1ab90291ccf8b2 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 25 Feb 2022 11:23:50 +0800 Subject: [PATCH 183/212] =?UTF-8?q?=E5=AF=BC4=E4=B8=AA=E6=9C=88=E6=95=B0?= =?UTF-8?q?=E6=8D=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/ccbevaluation/ExportOracleEvaluationData.groovy | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 603a00f36..61e8661f5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -307,7 +307,8 @@ class ExportOracleEvaluationData { // // for(def city : citys) { // def cityCode = cityCodeDataMap.get(city) - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; +// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] From e7410d4e9121ea2d5974cd4fb61b8be571836d90 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 25 Feb 2022 11:35:18 +0800 Subject: [PATCH 184/212] =?UTF-8?q?=E5=8F=AA=E8=BF=90=E8=A1=8C=E4=B8=80?= =?UTF-8?q?=E4=B8=AA?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ExportOracleEvaluationData.groovy | 28 +++++------ .../gj/location/DistrictLocationUpdate.groovy | 49 +++++++++++++++++-- 2 files changed, 59 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index 61e8661f5..d2c1656b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -20,23 +20,23 @@ class ExportOracleEvaluationData { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //计算结果相关 - exportAvmEvaluationGJNB(curMonth) - exportAVMEvaluationItemGJNB(curMonth) - - exportCityPrice(curMonth) - exportRegionPrice(curMonth) - - exportCommunityBasePriceGj(curMonth)//已合下面的合并 - exportCommunityBasePrice(curMonth) +// exportAvmEvaluationGJNB(curMonth) +// exportAVMEvaluationItemGJNB(curMonth) +// +// exportCityPrice(curMonth) +// exportRegionPrice(curMonth) +// +// exportCommunityBasePriceGj(curMonth)//已合下面的合并 +// exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) - exportCommCaseDetailGj(curMonth) - - def curDate = curMonth.toString()+"28" - SSHHelper jschCommand = new SSHHelper(); - System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// exportCommCaseDetailGj(curMonth) +// +// def curDate = curMonth.toString()+"28" +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); //ok文件 - exportOkFile() +// exportOkFile() } def exportAvmEvaluationGJNB(yearMonth){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy index 90712ca06..362bf2afb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/location/DistrictLocationUpdate.groovy @@ -1,12 +1,16 @@ package com.fangjia.script.gj.location import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.GeocodeUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.gj.OpenApiUtils import com.fangjia.script.gj.district.NameProcess import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject +import org.codehaus.jackson.sym.NameN import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet @@ -26,6 +30,7 @@ class DistrictLocationUpdate { def task = new DistrictLocationUpdate() // task.run() task.mapMatch() +// task.updateLocationToDB() } def run(){ def citys = ["福州"]//CommonDatautils.findAllCity() @@ -51,7 +56,7 @@ class DistrictLocationUpdate { def mapMatch(){ def index = 0 ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d-> - if(!d['H'] && d['C']=="凤凰小区"){ + if(!d['H'] && d['C']=="华侨城龙居御景二期"){ index++ // def gdData = FangjiaClientUtils.getMapData(d['A'], d['B'],d['C'], d['D'],"住宅", "高德") // println d['A']+" "+d['B']+" "+d['C'] @@ -100,6 +105,34 @@ class DistrictLocationUpdate { },0) } + def updateLocationToDB(){ + ExcelUtils.modify("D:\\数据\\小区\\经纬度\\福州V3.xlsx",{d-> + if(d['N'] && d['N']>1000){ + if(d['M']>=0.4 && d['K']){ + def query = new BasicDBObject() + query.put("city", d['A']) + query.put("region", d['B']) + query.put("name", d['C']) + + def udata = new BasicDBObject() + udata.put("m_date", new Date()) + + Long lng0 = d['K'].toString().toDouble()*10000000000L + Long lat0 = d['L'].toString().toDouble()*10000000000L + udata.put("lat", lat0) + udata.put("lng", lng0 ) + + def gmap = GeocodeUtils.getGaodeToBaidu(d['K'].toString().toDouble(), d['L'].toString().toDouble()) + Long lng2 = gmap.get("lng")*10000000000L + Long lat2 = gmap.get("lat")*10000000000L + udata.put("lat2", lat2) + udata.put("lng2", lng2) + DbUtils.seaweed.update(query, new BasicDBObject('$set', udata)) + } + } + },0) + } + def findStandardObject2(def city, def region, def name, def address, def rets) { if (rets) { @@ -124,7 +157,6 @@ class DistrictLocationUpdate { def maxNameScore = 0 for(def n : names){ double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); - score1 = Eval.me(score1.toString()) if(score1 && score1>0){ def nameScore = (score1 * 4)/10 if(nameScore > maxNameScore){ @@ -134,7 +166,12 @@ class DistrictLocationUpdate { } if(maxNameScore){ - maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + } score = maxNameScore + score; } @@ -142,7 +179,11 @@ class DistrictLocationUpdate { if(mapObj.address && address){ double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); if(addressScore && addressScore>0){ - addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } score = addressScore*4/10 + score } } From 2773c99b2b13a8f97e45d53eba1109881017592b Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 25 Feb 2022 14:15:10 +0800 Subject: [PATCH 185/212] update --- .../com/fangjia/core/util/BaiDuApiUtils.java | 2 +- .../ExportOracleEvaluationData.groovy | 34 +++++++++---------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index 4f59fe761..354e4c4ff 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -13,7 +13,7 @@ import java.util.*; */ public class BaiDuApiUtils { - private static String AK = "9E99D3540e4dafb7894f7b9fe27fa6b5"; + private static String AK = "ajgWIBd9rYsXed1hcDQqYvkX"; public static String[] AKS = new String[]{"hj5KqPjaGA3uivtg8uS8XfAnj9X1kPMr","xjFg7DfDqvQleIZasM3ph8Zi2Lw8kgCg", "4pb6fzrWIzvibDu8L1o8DGoY2h1wRIUq","ZzX6mRFfE5GKn4cpUp24Q3OyUIHnfQAR","79ac4ea49e7dedbf80b0ecd475d07784", "9E99D3540e4dafb7894f7b9fe27fa6b5","W3M5uHfcrmlAdhQjf6v93LCI", "C8bb035aacf602551e829e4c64575224", diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy index d2c1656b3..edc128cf1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleEvaluationData.groovy @@ -20,23 +20,23 @@ class ExportOracleEvaluationData { def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), 0, "yyyyMM")); //计算结果相关 -// exportAvmEvaluationGJNB(curMonth) -// exportAVMEvaluationItemGJNB(curMonth) -// -// exportCityPrice(curMonth) -// exportRegionPrice(curMonth) -// -// exportCommunityBasePriceGj(curMonth)//已合下面的合并 -// exportCommunityBasePrice(curMonth) + exportAvmEvaluationGJNB(curMonth) + exportAVMEvaluationItemGJNB(curMonth) + + exportCityPrice(curMonth) + exportRegionPrice(curMonth) + + exportCommunityBasePriceGj(curMonth)//已合下面的合并 + exportCommunityBasePrice(curMonth) exportCommunityPdescribeGj(curMonth) -// exportCommCaseDetailGj(curMonth) -// -// def curDate = curMonth.toString()+"28" -// SSHHelper jschCommand = new SSHHelper(); -// System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); - //ok文件 -// exportOkFile() + exportCommCaseDetailGj(curMonth) + + def curDate = curMonth.toString()+"28" + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb/output/gz_package.sh ${curDate}")); +// ok文件 + exportOkFile() } def exportAvmEvaluationGJNB(yearMonth){ @@ -307,8 +307,8 @@ class ExportOracleEvaluationData { // // for(def city : citys) { // def cityCode = cityCodeDataMap.get(city) -// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; - def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; + def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time=${month}"; +// def sql = "select ID,COMMUNITY_ID,CITYID ,CITY_NAME,DISTRICT_CODE ,DISTRICT_NAME,COMMUNITY_NAME,PQ_NAME,BASE_PRICE,CONFIDENCE_LEVEL,CNT,MIN_PRICE,MEDIANPRICE,MEANPRICE,MAX_PRICE,TIME,FIRSTDATAT,THIRDDATA,RELATIVEERROR,SKEWNESS,PQ_PRICE_RANGE,DISTRICT_PRICE_RANGE ,DESCRIPTION,PRIV_BASE_PRICE,IS_ARTTFICIAL,PRIV_FINAL_PRICE,to_char(CREATE_DATE,'yyyy-mm-dd') as CREATE_DATE,PRICE,PRICE_SOURCE,CASE_MD_PRICE,CASE_AVG_PRICE,INDEX_LABEL,MAIN_PROPERTY,COMMTYPE,NEW_DEAL_PRICE,to_char(NEW_DEAL_DATE,'yyyy-mm-dd') as NEW_DEAL_DATE,PRICE_MOM_RATIO,MAX_MOM_RATIO,PRICE_YOY_RATIO ,SATUR_RATIO,NEIGHBOR_PRICE from community_p_describe_gj where time in (202110,202111,202112,202201)"; def allRows = [] sqlInstance.eachRow(sql, { r -> def data = [] From ec415bed82f53b9820376c3e26803689b5db6cb6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 28 Feb 2022 11:17:21 +0800 Subject: [PATCH 186/212] update --- .../ccbevaluation/ExportOracleData.groovy | 28 +++--- .../fangjia/script/ccbevaluation/Test.groovy | 41 ++++++--- .../com/fangjia/script/gj/TestEmail.groovy | 4 +- .../gj/test/CrawlDistrictIdSelect.groovy | 89 +++++++++++++------ .../fangjia/script/jms/EmailConsumer.groovy | 4 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 2 +- 6 files changed, 110 insertions(+), 58 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 20e49d87c..e499c32ab 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,21 +25,21 @@ class ExportOracleData { def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //不会变 - exportEvaluationRateData() - exportBigdataPoiData() - exportBigdataDistrictData() - - exportGuidedPrice()//随人工改变 - exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 - -// //处理小区情况 - exportBigdataCommunity() - exportCommunityStaging() - exportCommunityBuileType() - - exportThirdPartyPrice2(lastMonth) +// exportEvaluationRateData() +// exportBigdataPoiData() +// exportBigdataDistrictData() +// +// exportGuidedPrice()//随人工改变 +// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 +// +//// //处理小区情况 +// exportBigdataCommunity() +// exportCommunityStaging() +// exportCommunityBuileType() +// +// exportThirdPartyPrice2(lastMonth) exportBigdataForsale("2022-01-19") - exportBigdataDeal("2022-01-01") +// exportBigdataDeal("2022-01-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 146c1f07b..0b63403ce 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -224,18 +224,18 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def } -def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" -def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") -def testMap = findCaluateCityPPE(sqlInstance236, 202202) -ExcelUtils.modify(fileName,{d-> - if(d['C']){ - def oldcount = dealMap.get(d['C']) - d['V'] = oldcount - def casecount = testMap.get(d['C']) - d['W'] = casecount - println d - } -},0) +//def fileName = "D:\\数据\\建行\\房产估值\\成交\\成交数据_1222 - 副本.xlsx" +//def dealMap = findDealCount(sqlInstance236, "2022-01-01", "2022-02-01") +//def testMap = findCaluateCityPPE(sqlInstance236, 202202) +//ExcelUtils.modify(fileName,{d-> +// if(d['C']){ +// def oldcount = dealMap.get(d['C']) +// d['V'] = oldcount +// def casecount = testMap.get(d['C']) +// d['W'] = casecount +// println d +// } +//},0) //ExcelUtils.multiThreadingModify("D:\\数据\\小区\\5个城市小区列表0723(1).xlsx",{d-> // if(d['I'] && d['K']){ @@ -782,6 +782,23 @@ def getStreetInfo(def city, def glocation){ //} //ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") +def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) + +def keys = [] +for(def d : datas2){ + keys << d['A']+"##"+d['C']+"##"+d['B'] +} + +def index = 0 +for(def d : datas1){ + def k = d['A']+"##"+d['C']+"##"+d['B'] + if(!keys.contains(k)){ + index ++ + println index +" "+k + } +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy index a0921b0c4..4e66c4284 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TestEmail.groovy @@ -7,11 +7,11 @@ email.setSSL(true); email.setHostName("smtp.exmail.qq.com"); email.setSslSmtpPort("465"); email.setCharset("GBK") -email.setAuthentication("service@fangjia.com", "Goojia123456"); //用户名和密码 +email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); //用户名和密码 email.addTo("gaojuan@fangjia.com"); //接收方 email.setFrom("service@fangjia.com"); //发送方 email.setSubject("test"); //标题 -email.setContent("你?", "text/html"); +email.setContent("测试", "text/html"); //email.setHostName("smtp.exmail.qq.com"); //email.setSSL(true); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy index 71142969a..0470fa957 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/test/CrawlDistrictIdSelect.groovy @@ -37,7 +37,10 @@ class CrawlDistrictIdSelect { } def run(){ - def datas = findCrawlData() + def source = "58同城" + def root = "58.com" + def shortCityMap = findCitySeeds(root) + def datas = findCrawlData(source, shortCityMap, root) def list = [] datas.each {k,v-> if(v){ @@ -58,15 +61,32 @@ class CrawlDistrictIdSelect { } } - def findCrawlData(){ + def findCitySeeds(root){ def q = new BasicDBObject() - q.put("city","北京") - q.put("source", "搜房") + q.put("root", root) + q.put("city", "福州") + + def dbcursor = districtseedsColl.find(q) + def cityMap = [:] + for(def d : dbcursor){ + if(!cityMap.keySet().contains(d.city)){ + cityMap.put(d.city, d.city_short) + } + } + return cityMap + + } + + def findCrawlData(def source, shortCityMap, def root){ + def q = new BasicDBObject() + q.put("city","福州") + q.put("source", source) q.put("s_date",202112) def dbcursor = thirdPartyColl.find(q).batchSize(10000).setOptions(Bytes.QUERYOPTION_NOTIMEOUT) def retMap = [:] - def prefix = "esf.fang.com/loupan/" + def prefix = "xiaoqu/" def keyList =new HashSet() + def index = 0 for(def db : dbcursor){ def url = null if(!db.url){ @@ -76,36 +96,42 @@ class CrawlDistrictIdSelect { }else{ url = db.url } - if(url && url.contains(prefix) ){ - def district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.indexOf(".htm")) + + if(url){ + def district_id = "" + if(url.contains(prefix)){ + district_id = url.substring(url.indexOf(prefix)+prefix.length(),url.toString().length()-1) + }else{ + district_id = url; + } if(district_id=="0"){ continue } - def shortCity = url.substring(url.indexOf("//")+2, url.indexOf(prefix)) - if(!shortCity){ - shortCity = "bj" - } - def key = shortCity +"#"+district_id - def ids = retMap.get(key) + def ids = retMap.get(db.city) if(!ids){ ids = [] - retMap.put(key, ids) + retMap.put(db.city, ids) } - - if(!keyList.contains(key)){ - def map = new BasicDBObject() - map.root = "fang.com" - map.city_short= shortCity - map.city = db.city - map.region = db.region - map.district_id = district_id - map.source_url = url - ids << map - - keyList << key + if(!keyList.contains(db.city+"#"+district_id)){ + def count = findIsExists(db.city, district_id, root) + if(!count){ + def map = new BasicDBObject() + map.root = "58.com" + map.city_short= shortCityMap.get(db.city) + map.city = db.city + map.region = db.region + map.district_id = district_id + map.source_url = url + map.name = db.name + ids << map + }else{ + index++ + println "[${index}]已存在:"+db.city +" "+db.name + } + keyList << db.city+"#"+district_id }else{ - println "重复:"+key + println "重复:"+db.city+"#"+db.name } }else{ println url +" 链接不合要求" @@ -113,4 +139,13 @@ class CrawlDistrictIdSelect { } return retMap } + + def findIsExists(city, district_id, root){ + def q = new BasicDBObject() + q.put("city",city) + q.put("district_id", district_id) + q.put("root",root) + def count = districtseedsColl.count(q) + return count + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy index 21f508609..f43bda14e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jms/EmailConsumer.groovy @@ -133,7 +133,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "G00jia123456"); + email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com");//发送方 email.setSubject(mail.subject); //标题 @@ -168,7 +168,7 @@ public class EmailConsumer extends BaseBatch { email.setHostName(EMAIL_SERVER); email.setSslSmtpPort("465"); email.setCharset("GBK") - email.setAuthentication("service@fangjia.com", "G00jia123456"); + email.setAuthentication("service@fangjia.com", "bRvvVvic8HVgHkCi"); email.addTo(mail.receiverEmail); email.setFrom("service@fangjia.com"); email.setSubject(mail.subject); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index bc691499b..5fb2fef63 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -181,7 +181,7 @@ class Esf58Crawl extends BaseBatch{ // "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", // "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" def cityList = [ - "重庆":"cq|37","保定":"bd|424","信阳":"xy|8694","南京":"nj|172","南宁":"nn|845","厦门":"xm|606","青岛":"qd|122","哈尔滨":"hrb|202","天津":"tj|18","太原":"ty|740","宁德":"nd|7951","宜昌":"yc|858","成都":"cd|102","惠州":"huizhou|722","扬州":"yz|637","昆明":"km|541","杭州":"hz|79","武汉":"wh|158","泰州":"taizhou|693","洛阳":"luoyang|556","深圳":"sz|4","石家庄":"sjz|241","贵阳":"gy|2015","连云港":"lyg|2049","邯郸":"hd|572", + "福州":"fz|304" ] static main(args) { From 23510ecdfb15f7dfc4818f310768f0da096f66d7 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 28 Feb 2022 11:36:14 +0800 Subject: [PATCH 187/212] update --- .../ccbevaluation/ExportOracleData.groovy | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index e499c32ab..20e49d87c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -25,21 +25,21 @@ class ExportOracleData { def lastMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); //不会变 -// exportEvaluationRateData() -// exportBigdataPoiData() -// exportBigdataDistrictData() -// -// exportGuidedPrice()//随人工改变 -// exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 -// -//// //处理小区情况 -// exportBigdataCommunity() -// exportCommunityStaging() -// exportCommunityBuileType() -// -// exportThirdPartyPrice2(lastMonth) + exportEvaluationRateData() + exportBigdataPoiData() + exportBigdataDistrictData() + + exportGuidedPrice()//随人工改变 + exportCommunityGuidedPrice(lastMonth.toString()+"28")//随政策 + +// //处理小区情况 + exportBigdataCommunity() + exportCommunityStaging() + exportCommunityBuileType() + + exportThirdPartyPrice2(lastMonth) exportBigdataForsale("2022-01-19") -// exportBigdataDeal("2022-01-01") + exportBigdataDeal("2022-01-01") //计算结果相关 From 0888fd2eea4d08563d5026bce025ec5f566cba34 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 1 Mar 2022 15:39:15 +0800 Subject: [PATCH 188/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E8=9E=8D=E5=90=88=E7=9A=84=E6=B5=8B=E8=AF=95=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/CcbCommonUtils.groovy | 23 + .../ccbevaluation/CcbFtpTestUtils.groovy | 65 ++ .../ronghe/BigdataCommunityGJRHBatch.groovy | 101 +-- .../ronghe/ExportDistrictRongHeTest.groovy | 742 ++++++++++++++++++ .../script/commData/CommonDatautils.groovy | 1 + .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- 6 files changed, 874 insertions(+), 62 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy index 78153a5f0..8a8b4a7dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbCommonUtils.groovy @@ -488,4 +488,27 @@ class CcbCommonUtils { DEAL_PRICE_COLL.update(new BasicDBObject("_id",id), new BasicDBObject('$set',udata)) } + public static def findDistrictSeqno(){ + def sql = "select cityid,commid,seqno from bigdata_position_seqno"; + def map = new HashMap() + sqlInstance.eachRow(sql,{r-> + def detailMap = map.get(r.cityid) + if(!detailMap){ + detailMap = [:] + map.put(r.cityid, detailMap) + } + detailMap.put(r.commid, r.seqno) + }); + return map + } + + public static def findMaxSeqno(def cityid){ + def sql = "select max(seqno) as max_seqno from bigdata_position_seqno where cityid='${cityid}'" + def maxSeqno = null; + sqlInstance.eachRow(sql,{r-> + maxSeqno = r.max_seqno + }); + return maxSeqno + } + } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy new file mode 100644 index 000000000..4190d947b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -0,0 +1,65 @@ +package com.fangjia.script.ccbevaluation + +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.jiangnanblank.SFTP +import com.jcraft.jsch.ChannelSftp +import org.apache.commons.codec.digest.DigestUtils + +class CcbFtpTestUtils { + + static def ftpServer = "192.168.0.86" + static def ftpPort = 17495 + static def ftpUser = "ccb" + static def ftpPassword = "ccb!@#" + static def ftpPath = "/home/ccb/output/" + + static def ftpPath_rh = "/home/ccb_rh/output/" + + static SFTP sftp = new SFTP(); + static ChannelSftp sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); + + + static def ftpUpload(type,outputPath_dist, month){ + //上传到ftp + sftpChannel = reConnect(); + if(type && type=="rh"){ + sftp.upload(ftpPath_rh+ month+"/", outputPath_dist, sftpChannel); + }else{ + sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); + } + + } + + static def reConnect(){ + if(sftpChannel == null){ + println "sftpChannel 为空,需重连....." + if(sftp == null){ + sftp = new SFTP(); + } + sftpChannel = sftp.connect(ftpServer, ftpPort, ftpUser, ftpPassword); + }else{ + println "sftpChannel 正常....." + } + return sftpChannel; + } + + static def uploadSha512Data(def sb, def dataCodeFileName, def charset){ + String sha512HexStr= DigestUtils.sha512Hex(sb.getBytes())//生成加密码码 + DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件 + ftpUpload(dataCodeFileName)//上传加密码文件到FTP + } + + static def uploadSha512Data2(def dataFileName, def dataCodeFileName, def charset){ + + def data = new FileInputStream(new File(dataFileName)) + String sha512HexStr= DigestUtils.sha512Hex(data)//生成加密码码 + DataHandleUtils.outPutFile(dataCodeFileName, sha512HexStr,charset);//生成加密码文件 + ftpUpload(dataCodeFileName)//上传加密码文件到FTP + } + + + static main(args) { + + + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index b25c8bc10..6409c9795 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -55,6 +55,8 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // addBigdataDistrictFJW() + def citySeqnoMap = CcbCommonUtils.findDistrictSeqno(); + def index = 0 for (def city : citys) { index++ @@ -88,20 +90,20 @@ class BigdataCommunityGJRHBatch extends BaseBatch { dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) } } - println index +" "+city +" 小区加载完成 "+districts.size() - addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 1.小区表完成" - addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 2.小区建筑结构表" - addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 3.小区房屋类型表" - addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 4.小区建筑年代表" - addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 5.小区费用表" - addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) - println index +" "+city +" 6.小区房屋用途表" - addBigdataPositionGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 小区加载完成 "+districts.size() +// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 1.小区表完成" +// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 2.小区建筑结构表" +// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 3.小区房屋类型表" +// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 4.小区建筑年代表" +// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 5.小区费用表" +// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) +// println index +" "+city +" 6.小区房屋用途表" + addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) println index +" "+ city + " 7.地理位置表" // addDealFieldInfo(distMap, cityCodeDataMap.get(city));//修改成交记录housestyle 一定要运行 @@ -552,18 +554,17 @@ class BigdataCommunityGJRHBatch extends BaseBatch { // } } - def addBigdataPositionGJ(districts, def cityid){ + def addBigdataPositionGJ(districts, def cityid, citySeqnoMap){ def rets = [] - def index = 0 for(def d : districts) { - index++ def ret = [] + def seqno = findSeqno(d.cityid, d.commid, citySeqnoMap); ret << d.cityid ret << "5" ret << d.commid ret << "1" - ret << index + ret << seqno ret << d.lng ret << d.lat ret << "" @@ -589,49 +590,29 @@ class BigdataCommunityGJRHBatch extends BaseBatch { def headers = [] FileUtils.batchCreateFile(rets, fileName, headers, "utf-8", true); println "BIGDATA_POSITION_FJW 数据文件输出完成......" + } -// sqlInstance.executeUpdate("delete from BIGDATA_POSITION_FJW where CITYID='${cityid}'") -// -// def sqls = [] -// def index = 1 -// for(def d : districts){ -// def sql = "INSERT INTO BIGDATA_POSITION_FJW (CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,CREATETIME,UPDATETIME,SOURCECODE) VALUES ('${d.cityid}','5', '${d.commid}','1',${index},${d.lng},${d.lat},null,${d.remark}," -// index++ -// if(d.createtime){ -// sql = sql + "to_date('${d.createtime}','yyyy-mm-dd hh24:mi:ss')," -// }else{ -// sql = sql + "null," -// } -// if(d.updatetime){ -// sql = sql + "to_date('${d.updatetime}','yyyy-mm-dd hh24:mi:ss'),'${d.sourcecode}')" -// }else{ -// sql = sql + "null,'${d.sourcecode}')" -// } -// sqls << sql -// -// if (sqls.size() >= batchsize) { -// try { -// sqlInstance.withBatch(batchsize, { stmt -> -// sqls.each { def s -> -// stmt.addBatch s -// } -// }) -// -// } catch (Exception e) { -// e.printStackTrace() -// } -// sqls.clear() -// } -// -// } -// -// if (sqls.size() > 0) { -// sqlInstance.withBatch(100, { stmt -> -// sqls.each { def s -> -// stmt.addBatch s -// } -// }) -// } + def findSeqno(cityid, commid, citySeqnoMap){ + def detailMap = citySeqnoMap.get(cityid) + if(detailMap){ + def seqno = detailMap.get(commid) + if(seqno){ + return seqno + }else{ + seqno = CcbCommonUtils.findMaxSeqno(cityid) + if(seqno){ + seqno = seqno + 1 + def createtime = DateUtils.formatDateByPattern(new Date(), "yyyy-MM-dd HH:mm:ss") + def insertSql = "INSERT INTO bigdata_position_seqno(CITYID,COMMID,SEQNO,CREATETIME) VALUES ('${cityid}','${commid}',${seqno},to_date('${createtime}','yyyy-mm-dd hh24:mi:ss'))" + sqlInstance.withBatch(batchsize, { stmt -> + stmt.addBatch insertSql + }) + return seqno + } + } + }else{ + println cityid+" 没有该城市的小区序号" + } } def addBigdataDistrictFJW(){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy new file mode 100644 index 000000000..0c1c0c558 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -0,0 +1,742 @@ +package com.fangjia.script.ccbevaluation.ronghe + +import com.fangjia.script.BaseBatch +import com.fangjia.script.BatchOwner +import com.fangjia.script.Watcher +import com.fangjia.script.ccbevaluation.CcbCommonUtils +import com.fangjia.script.ccbevaluation.CcbFtpTestUtils + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.ccb.FileUtils +import com.fangjia.script.util.DateUtils +import groovy.sql.Sql + +import java.sql.ResultSet +import java.sql.ResultSetMetaData + +@BatchOwner(Watcher.GaoJuan) +class ExportDistrictRongHeTest extends BaseBatch{ + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + static main(args) { + runBatch(new ExportDistrictRongHeTest(), args); + } + + @Override + void process(Object args) { + def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) + + exportBigdataCommunityGJRH() + exportBigdataBuildingFJW() + exportBigdataCommfeeGj() + + exportBigdataCommbuildstruGJ() + exportBigdataCommhousetypeGJ() + exportBigdataCommbuildyearGj() + exportBigdataCommhouseuseGJ() + exportBigdataPositionGJ() + exportBigdataDistrictFJW() + exportBigdataAliasFJW() + + exportBigdataBuildprojFjw() + exportBigdataCommlayoutpicFjw() + exportEvaluateRate() + + SSHHelper jschCommand = new SSHHelper(); + System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); + + exportOk(curDate) + + } + +// +// def runByCity(){ +// def citys = CcbCommonUtils.citys; +// def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() +// +// +//// //检查ok文件是否上传 +//// exportAllBigdataDealFJW() +//// for(def city : citys) { +//// def cityCode = cityCodeDataMap.get(city) +//// +// //按天导(一个文件夹下) +// //exportBigdataDealFJW(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) +// //按天导(每天一个文件夹) +// exportBigdataDealFJW2(DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date(),-2), "yyyy-MM-dd")) +// +//// exportBigdataCommunityGJRH(cityCode) +//// exportBigdataBuildingFJW(cityCode) +//// exportBigdataCommfeeGj(cityCode) +//// +//// exportBigdataCommbuildstruGJ(cityCode) +//// exportBigdataCommhousetypeGJ(cityCode) +//// exportBigdataCommbuildyearGj(cityCode) +//// exportBigdataCommhouseuseGJ(cityCode) +//// exportBigdataPositionGJ(cityCode) +//// exportBigdataDistrictFJW(cityCode) +//// exportBigdataAliasFJW(cityCode) +//// +//// exportBigdataBuildprojFjw(cityCode) +//// exportBigdataCommlayoutpicFjw(cityCode) +//// } +//// exportEvaluateRate() +// +//// +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") +// SSHHelper jschCommand = new SSHHelper(); +// System.out.println(jschCommand.executeCommand("sh /home/ccb_rh/output/rh_package.sh ${curDate}")); +// +// exportOk() +// } + + def exportBigdataCommunityGJRH(cityCode){ + def sql = "select CITYID,DISTID,COMMID,COMMNAME,ADDRESS,LANDAREA,BUILDAREA,GREENINGRATE,PLOTRATIO,PARKINGCNT,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,COMMTYPE,COMMALIAS,STREET,STREETNO,IFNEW,INTRODUCTION,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,SOURCECODE from BIGDATA_COMMUNITY_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMMNAME + data << r.COMMALIAS + data << r.DISTID + data << r.STREET + data << r.STREETNO + data << r.ADDRESS + data << r.LANDAREA + data << r.BUILDAREA + data << r.COMMTYPE + data << r.IFNEW + data << r.GREENINGRATE + data << r.PLOTRATIO + data << r.PARKINGCNT + data << '' + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMUNITY_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + println "BIGDATA_COMMUNITY_FJW 数据文件输出完成......" + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMUNITY_FJW 上传Ftp完成......" + } + def exportBigdataCommbuildstruGJ(){ + def sql = "select CITYID,COMMID,BUILDSTRU,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDSTRU_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDSTRU + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDSTRU_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMBUILDSTRU_FJW 上传Ftp完成......" + } + + def exportBigdataCommhousetypeGJ(cityCode){ + def sql = "select CITYID,COMMID,HOUSETYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSETYPE_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSETYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSETYPE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMHOUSETYPE_FJW 上传Ftp完成......" + } + + def exportBigdataCommbuildyearGj(){ + def sql = "select CITYID,COMMID,BUILDYEAR,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMBUILDYEAR_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.BUILDYEAR + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMBUILDYEAR_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "BIGDATA_COMMBUILDYEAR_FJW 上传Ftp完成......" + } + + def exportBigdataCommfeeGj(){ + def sql = "select CITYID,COMMID,COMPID,FEE,HOUSEUSE,BUILDTYPE,FEETYPE,NOTE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMFEE_FJW";//where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.COMPID + data << r.FEE + data << r.HOUSEUSE + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.FEETYPE + data << r.NOTE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMFEE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMFEE_FJW 上传Ftp完成......" + } + + + def exportBigdataCommhouseuseGJ(){ + def sql = "select CITYID,COMMID,HOUSEUSE,HOUSECNT,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_COMMHOUSEUSE_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.COMMID + data << r.HOUSEUSE + data << r.HOUSECNT + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMHOUSEUSE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMHOUSEUSE_FJW 上传Ftp完成......" + } + + def exportBigdataPositionGJ(){ + def sql = "select CITYID,OBJECTTYPE,OBJECTID,POSITIONTYPE,SEQNO,LONGITUDE,LATITUDE,ALTITUDE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_POSITION_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.OBJECTTYPE + data << r.OBJECTID + data << r.POSITIONTYPE + data << r.SEQNO + data << r.LONGITUDE + data << r.LATITUDE + data << r.ALTITUDE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_POSITION_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_POSITION_FJW 上传Ftp完成......" + } + + def exportBigdataDistrictFJW(){ + def sql = "select DISTID,DISTNAME,DISTSPELL,CITYID,DISTCODE,SEQ,DISTTYPE,REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from BIGDATA_DISTRICT_FJW";// where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.DISTID + data << r.DISTNAME + data << r.DISTSPELL + data << r.CITYID + data << r.DISTCODE + data << r.SEQ + data << r.DISTTYPE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DISTRICT_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "BIGDATA_DISTRICT_FJW 上传Ftp完成......" + } + + def exportBigdataAliasFJW(){ + def sql = "select city_id, ca_id,address_id,address_name, status, exp_dt, version, is_key_name, data_source, description,alias_grade,addr_type_cd,ifexamine,ifdisplay from ADDRESS_ALIAS_FJW";// where city_id='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.city_id + data << r.ca_id + data << r.address_id + data << r.address_name + data << r.status + data << r.exp_dt + data << r.version + data << r.is_key_name + data << r.data_source + data << r.description + data << r.alias_grade + data << r.addr_type_cd + data << r.ifexamine + data << r.ifdisplay + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_ADDRESS_ALIAS_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "ADDRESS_ALIAS_FJW 上传Ftp完成......" + } + + def exportBigdataDealFJW(dealdate){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def stop = false + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def index = 0 + + def curStartDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()),"yyyy-MM-dd") + def curSndDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(new Date()),"yyyy-MM-dd") + + while(!stop){ + index++ + def curStartStr = curStartDateStr + + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curStartStr.replace("-","")}_0001.dat" + } + + def allRows = [] + if(index==1){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + } + if(curStartDateStr == curSndDateStr){ + stop = true + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh", fileName, curDate) + println "BIGDATA_DEAL_FJW 上传Ftp完成......${curStartStr}" + def curEndStr = DateUtils.add(curStartStr, 1) + curStartDateStr = curEndStr + } + } + + + def exportBigdataDealFJW2(curDate, dealdate){ + + def startDateStr = DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") + def endDateStr = DateUtils.formatDateByPattern(DateUtils.getLastDayOfMonth(DateUtils.parseFromTightYYYYMMDD(dealdate)), "yyyy-MM-dd") +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + + def allRows = [] + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_DEAL_FJW 上传Ftp完成......${endDateStr}" + } + + def exportAllBigdataDealFJW(){ + def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where DEALDATE + def data = [] + data << r.CITYID + data << r.DEALID + data << r.CITYNAME + data << r.DISTID + data << r.DISTNAME + data << r.STREET + data << r.STREETNO + data << r.COMMID + data << r.COMMNAME + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.UNITID + data << r.UNITNO + data << r.UNITNAME + data << r.HOUSEID + data << r.ADDRESS + data << r.BUILDYEAR + data << r.PROPTYPE + data << r.PROPYEARS + data << r.PERIODNUM + data << r.FLOORNUM + data << r.FLOORTYPE + data << r.FLOORCNT + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.BUILDAREA + data << r.LIVINGAREA + data << r.HOUSESTYLE + data << r.ROOMCNT + data << r.LIVINGROOMCNT + data << r.KITCHENCNT + data << r.BATHROOMCNT + data << r.HOUSEUSE + data << r.HOUSESTRU + data << r.HOUSEASPECT + data << r.FITMENT + data << r.EVALUATENO + data << r.DEALUNITPRICE + data << r.DEALTOTALPRICE + data << r.DECLAREPRICE + data << r.EVALUATEPRICE + data << r.CONTRACTNO + data << r.DEALDATE + data << r.DEALTYPE + data << r.PAYTYPE + data << r.DEALPERIOD + data << r.IFELEVATOR + data << r.COMPAREAGARDEN + data << r.COMPAREATERRACE + data << r.COMPAREABALCONY + data << r.COMPAREABAYWINDOW + data << r.COMPAREAATTIC + data << r.COMPAREAGARAGE + data << r.COMPAREABASEMENT + data << r.COMPAREAOTHER + data << r.DATASOURCE + data << r.AGENCYID + data << r.AGENCYNAME + data << r.REMARK + data << r.CREATETIME1 + data << r.UPDATETIME1 + data << r.SOURCECODE + data << r.WORKDATE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_DEAL_FJW_ADD_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + println "BIGDATA_DEAL_FJW 导出完成......" + } + + + def exportBigdataBuildingFJW(cityCode){ + def sql = "select CITYID, BUILDID, BUILDNO, BUILDNAME, BUILDALIAS, DISTID, COMMID, BUILDYEAR, BUILDTYPE, BUILDSTRU, FLOORCNT, UNITCNT, HOUSECNT, IFELEVATOR, ELEVATORCNT, ELEVATORHOUSE, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE, VALID_FLAG from BIGDATA_BUILDING_FJW";//where CITYID='${cityCode}' + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.CITYID + data << r.BUILDID + data << r.BUILDNO + data << r.BUILDNAME + data << r.BUILDALIAS + data << r.DISTID + data << r.COMMID + data << r.BUILDYEAR + if(r.BUILDTYPE){ + def arrs = r.BUILDTYPE.replace("[","").replace("]","").split(",") + data << arrs[0].trim() + }else{ + data << '' + } + data << r.BUILDSTRU + data << r.FLOORCNT + data << r.UNITCNT + data << r.HOUSECNT + data << r.IFELEVATOR + data << r.ELEVATORCNT + data << r.ELEVATORHOUSE + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + data << r.VALID_FLAG + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDING_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDING_FJW 上传Ftp完成......" + } + + + + def exportEvaluateRate(){ + def sql = "select ID, RATE_TYPE, RATE, RANK, CITY, REMARK,to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME,SOURCECODE from EVALUATE_RATE_FJW"; + def allRows = [] + sqlInstance.eachRow(sql,{r-> + def data = [] + data << r.ID + data << r.RATE_TYPE + data << r.RATE + data << r.RANK + data << r.CITY + data << r.REMARK + data << r.CREATETIME + data << r.UPDATETIME + data << r.SOURCECODE + allRows << data + }); + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_EVALUATE_RATE_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile(allRows, fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "EVALUATE_RATE_FJW 上传Ftp完成......" + } + + def exportBigdataBuildprojFjw(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_BUILDPROJ_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_BUILDPROJ_FJW 上传Ftp完成......" + } + + def exportBigdataCommlayoutpicFjw(){ + def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMM")+"10" + def fileName = "/ccb_rh/${curDate}/FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + if(CommonDatautils.isWindow()){ + fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\FJW_000000_000000_BIGDATA_COMMLAYOUTPIC_FJW_ALL_${curDate}_0001.dat" + } + FileUtils.batchCreateFile([], fileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",fileName, curDate) + println "BIGDATA_COMMLAYOUTPIC_FJW 上传Ftp完成......" + } + + def exportOk(curDate){ +// def curDate = DateUtils.formatDateByPattern(new Date(), "yyyyMMdd") + def dataCodeFileName = "/ccb_rh/${curDate}/${curDate}.ok" + if(CommonDatautils.isWindow()){ + dataCodeFileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}\\${curDate}.ok" + } + FileUtils.batchCreateFile([], dataCodeFileName, [], "utf-8", false); + CcbFtpTestUtils.ftpUpload("rh",dataCodeFileName, curDate) + } + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index a25ab36f0..a3e425055 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -829,6 +829,7 @@ class CommonDatautils { keys.put("lat", 1); keys.put("lng", 1); keys.put("name_cat", 1); + keys.put("finish_building_date", 1) DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 1dec5f73e..e0d8713b6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20220101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20220201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20220201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20220301", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 From 85b918acbe83be12f75693ef789e16a6e8fe6a1f Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 4 Mar 2022 11:15:49 +0800 Subject: [PATCH 189/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/poi/GetCommitteeFromExcel.groovy | 25 +++++++++++++++ .../shenjian/poi/GetLngLatFromExcel.groovy | 17 ++++++++++ .../shenjian/poi/GetTownshipFromExcel.groovy | 31 +++++++++++++++++++ 3 files changed, 73 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy new file mode 100644 index 000000000..8125bf1db --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCommitteeFromExcel.groovy @@ -0,0 +1,25 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + // 乡镇以下级政府及事业单位(居委会) + def types = "130106" + def city = item['D'] + def lng = item['M'] + def lat = item['N'] + if (StringUtils.isBlank(item['G'])) { + List pois = GaoDeUtils.searchNearDtoByPoi(city, types, lng, lat, 1000, "") + if (CollectionUtils.isNotEmpty(pois)) { + AmapPoi amapPoi = pois.get(0) + if (amapPoi.getCityname().startsWith(city)) { + item['G'] = amapPoi.getName() + item['S'] = '修改社区' + } + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy new file mode 100644 index 000000000..6b37becf8 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetLngLatFromExcel.groovy @@ -0,0 +1,17 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.core.util.GeoUtils +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import org.apache.commons.lang.StringUtils + +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V8.xlsx",{List item-> + + if (item['L'] != null) { + def location = GeoUtils.gcjToGps84(item['L'], item['K']) + if (location) { + item['N'] = location.lat + item['M'] = location.lng + println(item) + } + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy new file mode 100644 index 000000000..c8481a44a --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetTownshipFromExcel.groovy @@ -0,0 +1,31 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取国家标准居委街道 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def flag = item['S'] + def area = item['E'] + def name = item['G'] + if (StringUtils.isNotBlank(item['G']) && StringUtils.isNotBlank(area) && String && flag == '修改社区' && item['T'] == null) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", area + "区").append("name", name), new BasicDBObject()).limit(1).each { + obj -> + item['F'] = obj.street_name + item['L'] = obj.street_code + item['T'] = "补充街道正确" + } + } +}) From 38c28bc2e9484f5ab40206f05dd3b550bee12b40 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 4 Mar 2022 17:27:25 +0800 Subject: [PATCH 190/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/com/fangjia/core/util/DataUtil.java | 19 +++++ .../FillTownshipAndCommunityFromExcel.groovy | 39 +++++++++ .../shenjian/address/GetAddressDetail.groovy | 79 +++++++++++++++++++ .../temp/shenjian/poi/GetCodeFromExcel.groovy | 48 +++++++++++ 4 files changed, 185 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java index 93218b265..c681c113d 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/DataUtil.java @@ -1435,6 +1435,25 @@ public class DataUtil { return null; } + /** + * 匹配符合条件的字符串 + * + * @param str + * @param pattern + * @return + */ + public static List matchAll(String str, Pattern pattern) { + if (str == null) { + return null; + } + List list = new ArrayList(); + Matcher matcher = pattern.matcher(str); + while (matcher.find()) { + list.add(matcher.group()); + } + return list; + } + public static boolean isNumeric(String str) { try { Double.parseDouble(str); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy new file mode 100644 index 000000000..79cf910db --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy @@ -0,0 +1,39 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils +import org.apache.commons.lang.StringUtils + +/** + * 补充详细地址中缺失的乡镇街道与居委社区 + */ +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", { + List item -> + def area = item['F'] + def township = item['G'] + def community = item['H'] + String detailAddr = item['I'] + + // 不包含街道与居委 + if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && !detailAddr.contains(community) ) { + if (StringUtils.isNotBlank(area)) { + int index = detailAddr.indexOf(area + "区") + area.length() + 2 + item['K'] = detailAddr.substring(0, index) + township + community + + detailAddr.substring(index) + } + // 不包含社区 + } else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + detailAddr.contains(township) && !detailAddr.contains(community)) { + int index = detailAddr.indexOf(township) + township.length() + 1 + item['K'] = detailAddr.substring(0, index) + community + + detailAddr.substring(index) + } // 只不包含街道 + else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && + !detailAddr.contains(township) && detailAddr.contains(community)) { + if (StringUtils.isNotBlank(area)) { + int index = detailAddr.indexOf(area + "区") + area.length() + 2 + item['K'] = detailAddr.substring(0, index) + township + + detailAddr.substring(index) + } + } +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy new file mode 100644 index 000000000..b77b6d36c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy @@ -0,0 +1,79 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.core.util.DataUtil +import com.fangjia.script.util.ExcelUtils +import net.sf.json.JSONObject +import org.apache.commons.lang.StringUtils + +import java.util.regex.Pattern + +//ExcelUtils.removeEmptyRow("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V10(优化补充社区冗余信息).xlsx") +ners = ["houseno", "cellno", "floorno", "roomno"] +Pattern GROUP_PATTERN = Pattern.compile("[一二三四五六七八九十甲乙丙丁\\d]+组") +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx",{List item-> + + if (StringUtils.isNotBlank(item["J"])) { + //item["I"] = item["J"] + //item["W"] = "详细地址替换" + } +// def addr = item["B"].toString() +// def detailAddr = item["I"]?.toString() +// def pd = item["C"] +// item['J'] = "" +// if (StringUtils.isNotBlank(pd)) { +// int len = addr.lastIndexOf(pd) +// if (len > 3) { +// String aim = addr.substring(len - 3, len) +// int aimIndex = detailAddr.lastIndexOf(aim) +// if (aimIndex > 0) { +// item['J'] = detailAddr.substring(0, aimIndex + 3) + pd +// } +// } +// } + +// if(detailAddr{ +// String group = DataUtil.match(pd, GROUP_PATTERN) +// if(group == null ){ +// return +// } +// def arabic_group = null +// if(!detailAddr.contains(group)){ +// arabic_group = CNNMFilter.qCNNumericToArabic(group) +// } +// if(group){ +// def flg = addr.substring(addr.indexOf(group) - 3, addr.indexOf(group)) +// flg = flg.replace("-", "") +// if(arabic_group != null){ +// group = arabic_group +// } +// if(!detailAddr.contains(flg + group)){ +// println(item) +// detailAddr = detailAddr.replace(group, '') +// def idx = detailAddr.lastIndexOf(flg) +// try{ +// detailAddr = detailAddr.substring(0, idx) + flg + group + detailAddr.substring(idx + flg.length()) +// item["j"] = detailAddr +// }catch(Exception e){ +// item["W"] ="组替换失败" +// } +// +// +// } +// } +// } + +// HttpRequest request = new HttpRequest() +// JSONObject result = request.getJsonContent("http://192.168.0.19:31080/FangJiaNlpService/ner",["address":addr], "utf-8") +// def index = Integer.MAX_VALUE +// for(JSONObject entity : result.entities){ +// if(entity.type in ners){ +// if(entity.start < index){ +// index = entity.start +// } +// } +// } +// if(index != Integer.MAX_VALUE){ +// item["C"] = addr.substring(index) +// println(item) +// } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy new file mode 100644 index 000000000..f466ed3be --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/GetCodeFromExcel.groovy @@ -0,0 +1,48 @@ +package com.fangjia.script.temp.shenjian.poi + +import com.fangjia.script.statistics.poi.AmapPoi +import com.fangjia.script.temp.shenjian.util.GaoDeUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.apache.commons.collections.CollectionUtils +import org.apache.commons.lang.StringUtils + +final DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) + +/** + * 获取省市区编码 + */ +com.fangjia.script.util.ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V9(补全省市区代码).xlsx",{List item-> + + def province = item['C'] + def city = item['D'] + def area = item['E'] + if (StringUtils.isBlank(province) && city == '重庆' && item['K'] == null) { + item['J'] = '5000000' + if (StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area)) { + villagesConn.find(new BasicDBObject("province_name", "重庆市") + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['J'] = obj.province_code + item['K'] = obj.area_code + } + } + } else if (StringUtils.isNotBlank(province) && StringUtils.isNotBlank(city) && StringUtils.isNotBlank(area) && item['K'] == null) { + villagesConn.find(new BasicDBObject("province_name", new BasicDBObject('$regex', '^' + province)) + .append("city_name", new BasicDBObject('$regex', '^' + city)) + .append("area_name", new BasicDBObject('$regex', '^' + area)), new BasicDBObject() ) + .limit(1).each {obj -> + item['I'] = obj.province_code + item['J'] = obj.city_code + item['K'] = obj.area_code + } + } + if (item['I'] != null) { + item['I'] = (String.valueOf(item['I']).replace(".0", "") + "000000").substring(0, 6); + } + if (item['J'] != null) { + item['J'] = (String.valueOf(item['J']).replace(".0", "") + "000000").substring(0, 6); + } +}) From dabc47389c1453d84b8c92a3aa88b3dd5b5b5f79 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 4 Mar 2022 18:13:23 +0800 Subject: [PATCH 191/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E9=87=8D=E5=BA=86?= =?UTF-8?q?=E5=86=9C=E5=95=86=E8=A1=8C=E6=B5=8B=E8=AF=95=E8=84=9A=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../address/FillNullTownshipAndComm.groovy | 23 ++++++++++++ .../FillTownshipAndCommunityFromExcel.groovy | 36 ++++++++++++++----- .../shenjian/address/GetAddressDetail.groovy | 5 ++- 3 files changed, 52 insertions(+), 12 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillNullTownshipAndComm.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillNullTownshipAndComm.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillNullTownshipAndComm.groovy new file mode 100644 index 000000000..13e8d4b8b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillNullTownshipAndComm.groovy @@ -0,0 +1,23 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils + +Map> map = new HashMap<>() +List> readList = ExcelUtils.read("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\(曹合1 )重庆农商行测试地址V8-1(街道、社区、街道代码补充)处理--新-芝.xlsx") +for (List result : readList) { + map.put(result.get(0), result) +} +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx", { + List item -> + def id = item['A'] + def township = item['G'] + def community = item['H'] + if (org.apache.commons.lang.StringUtils.isBlank(township)) { + item['G'] = map[id]['F'] + item['Y'] = '人工乡镇' + } + if (org.apache.commons.lang.StringUtils.isBlank(community)) { + item['H'] = map[id]['G'] + item['Z'] = '人工社区' + } +}) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy index 79cf910db..eca8609f9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillTownshipAndCommunityFromExcel.groovy @@ -13,27 +13,45 @@ ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识 def community = item['H'] String detailAddr = item['I'] + + if (detailAddr == '重庆市云阳县双江街道恒生场2-02') { + print("test") + } // 不包含街道与居委 if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && !detailAddr.contains(township) && !detailAddr.contains(community) ) { if (StringUtils.isNotBlank(area)) { - int index = detailAddr.indexOf(area + "区") + area.length() + 2 - item['K'] = detailAddr.substring(0, index) + township + community + - detailAddr.substring(index) + if (detailAddr.indexOf(area + "区") > 0) { + int index = detailAddr.indexOf(area + "区") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + community + + detailAddr.substring(index) + } else if (detailAddr.indexOf(area + "县") > 0) { + int index = detailAddr.indexOf(area + "县") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + community + + detailAddr.substring(index) + } } // 不包含社区 } else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && detailAddr.contains(township) && !detailAddr.contains(community)) { - int index = detailAddr.indexOf(township) + township.length() + 1 - item['K'] = detailAddr.substring(0, index) + community + - detailAddr.substring(index) + if (detailAddr.indexOf(township) > 0) { + int index = detailAddr.indexOf(township) + township.length() + item['K'] = detailAddr.substring(0, index) + community + + detailAddr.substring(index) + } } // 只不包含街道 else if (StringUtils.isNotBlank(township) && StringUtils.isNotBlank(community) && !detailAddr.contains(township) && detailAddr.contains(community)) { if (StringUtils.isNotBlank(area)) { - int index = detailAddr.indexOf(area + "区") + area.length() + 2 - item['K'] = detailAddr.substring(0, index) + township + - detailAddr.substring(index) + if (detailAddr.indexOf(area + "区") > 0) { + int index = detailAddr.indexOf(area + "区") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + + detailAddr.substring(index) + } else if (detailAddr.indexOf(area + "县") > 0) { + int index = detailAddr.indexOf(area + "县") + area.length() + 1 + item['K'] = detailAddr.substring(0, index) + township + + detailAddr.substring(index) + } } } }) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy index b77b6d36c..b8833c3fa 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/GetAddressDetail.groovy @@ -12,9 +12,8 @@ ners = ["houseno", "cellno", "floorno", "roomno"] Pattern GROUP_PATTERN = Pattern.compile("[一二三四五六七八九十甲乙丙丁\\d]+组") ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址V11(优化详细地址) .xlsx",{List item-> - if (StringUtils.isNotBlank(item["J"])) { - //item["I"] = item["J"] - //item["W"] = "详细地址替换" + if (StringUtils.isNotBlank(item["K"])) { + item["I"] = item["K"] } // def addr = item["B"].toString() // def detailAddr = item["I"]?.toString() From af74ec2cc61c4ef94a54da20e99e04ba92ea3843 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 8 Mar 2022 10:20:41 +0800 Subject: [PATCH 192/212] =?UTF-8?q?=E5=8F=AF=E8=B4=B7=E9=87=91=E9=A2=9D?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl index 2c92f982c..165012ff6 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhreport.ftl @@ -44,7 +44,7 @@ <#if role?? && role==17 && loanInfoMap??>

鎵�灞炲垎绫伙細${loanInfoMap.type!}绫�

鍙捶鎴愭暟锛�${loanInfoMap.loan_ratio!}鎴�

-

鍙捶閲戦锛�${(fjtEvaluation.totalPrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

+

鍙捶閲戦锛�${(bohaiEvaluationAudit.inquirePrice*loanInfoMap.loan_ratio*0.1/10000)?string('0.##')}涓囧厓

璐锋鍒╃巼锛�4.5%

<#else> From b2a92c6bf2cb67bd022f83e60ba9bd1ea9737d3b Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 8 Mar 2022 11:05:18 +0800 Subject: [PATCH 193/212] =?UTF-8?q?=E6=96=B0=E7=89=88=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=E6=A0=87=E5=87=86=E5=8C=96=E4=B8=8A=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 134 ++++++++++++++++++ .../src/main/resources/urlrewrite2.0.xml | 5 + 2 files changed, 139 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 907212d5f..07d121e77 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -106,6 +106,7 @@ public class AddressController { private HouseLoudongService houseLoudongService; private FjtEvaluationService fjtEvaluationService; private AddressMatchProxyService addressMatchProxyService; + private final static String AK = OpenConfig.getConfig().getBaiduAk(); private final static Logger LOGGER = LoggerFactory.getLogger(AddressController.class); @@ -1705,6 +1706,139 @@ public class AddressController { return mav; } + /** + * 地址标准化接口V2 + * + * @param model + * @return + */ + public Object standardV2Action(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : "重庆市"); + params.put("address", model.getAddress()); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 五级地址标准化接口 + * + * @param model + * @return + */ + public Object fiveDivisionStructureAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : "重庆市"); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + + /** + * 详细地址标准化接口 + * + * @param model + * @return + */ + public Object divisionStructureAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : "重庆市"); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/divisionStructure", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + public Object similarityAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { diff --git a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml index fb8d631db..0e434c5a4 100644 --- a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml +++ b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml @@ -33,5 +33,10 @@ /property/avgPrice + + + /address/standardV2 + + \ No newline at end of file From 73cc32931b8ba44a967b098d61bf60eb987aee07 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Tue, 8 Mar 2022 13:44:45 +0800 Subject: [PATCH 194/212] =?UTF-8?q?=E6=96=B0=E7=89=88=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E5=9D=80=E6=A0=87=E5=87=86=E5=8C=96=E4=B8=8A=E7=BA=BF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../src/main/java/com/fangjia/open/OpenConfig.java | 5 +++++ .../com/fangjia/open/controllers/AddressController.java | 9 ++++++--- .../src/main/resources/fangjia-open.properties | 3 ++- .../fangjia-open/src/main/resources/urlrewrite2.0.xml | 8 ++++---- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java b/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java index e1064e8cd..c13a28b36 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/OpenConfig.java @@ -38,6 +38,7 @@ public class OpenConfig extends Configuration { private final String ipdatfile; private final String baiduAk; + private final String defaultCity; private final boolean isOpenAllAuthority; @@ -54,6 +55,7 @@ public class OpenConfig extends Configuration { baiduAk = configuration.get("fangjia.baidu.ak"); isOpenAllAuthority = configuration.getBoolean("fangjia.isOpenAllAuthority", false); enabledAddressMatch = configuration.getBoolean("fangjia.addressMatch.enabled",false); + defaultCity = configuration.get("fangjia.default.city"); } public static OpenConfig getConfig() { @@ -134,4 +136,7 @@ public class OpenConfig extends Configuration { return enabledAddressMatch; } + public String getDefaultCity() { + return defaultCity; + } } diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 07d121e77..d11473fcc 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -89,6 +89,7 @@ import com.fangjia.service.mongo.fangjia.ServerAreaService; import com.fangjia.service.open.WaitCrawlKeywordsService; import com.fangjia.service.sql.FjtEvaluationService; import com.fangjia.service.web.AddressService; +import org.sothis.web.mvc.annotation.Param; @Authority(role = Role.USER) public class AddressController { @@ -118,6 +119,8 @@ public class AddressController { private static final String GRADE_NATION = "全国评分"; private static final String GRADE_CONSUME = "消费评分"; + public final static OpenConfig CONFIG = OpenConfig.getConfig(); + /** * @deprecated see parse2Action * @param model @@ -1712,7 +1715,7 @@ public class AddressController { * @param model * @return */ - public Object standardV2Action(AddesssMatchModel model) { + public Object standard2Action(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress())) { @@ -1723,7 +1726,7 @@ public class AddressController { HttpRequest request = new HttpRequest(); JSONObject json = null; Map params = new HashMap(); - params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : "重庆市"); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); params.put("address", model.getAddress()); try{ json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structure", params, "UTF-8"); @@ -1766,7 +1769,7 @@ public class AddressController { HttpRequest request = new HttpRequest(); JSONObject json = null; Map params = new HashMap(); - params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : "重庆市"); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); params.put("address", model.getAddress()); // 地址验证无效也输出标准化结果 params.put("flag", "true"); diff --git a/projects/fangjia-open/src/main/resources/fangjia-open.properties b/projects/fangjia-open/src/main/resources/fangjia-open.properties index aaf6af5ca..c7d4d526c 100644 --- a/projects/fangjia-open/src/main/resources/fangjia-open.properties +++ b/projects/fangjia-open/src/main/resources/fangjia-open.properties @@ -100,4 +100,5 @@ fangjia.jms.email.queue=email.queue fangjia.redis_nodes=192.168.0.79:6379,192.168.0.79:6380,192.168.0.79:6381,192.168.0.81:6382,192.168.0.81:6383,192.168.0.81:6384 -fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2 \ No newline at end of file +fangjia.fangjiatong.url = http://user.fangjia.com/fangjiatong2 +fangjia.default.city = \u91CD\u5E86\u5E02 \ No newline at end of file diff --git a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml index 0e434c5a4..ba0a21b49 100644 --- a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml +++ b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml @@ -1,8 +1,8 @@ - - + + @@ -33,9 +33,9 @@ /property/avgPrice - + - /address/standardV2 + /address/standard From 4cb8b4cd0f97e7d11ae06d381905f9651541ec8a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Mar 2022 11:10:03 +0800 Subject: [PATCH 195/212] =?UTF-8?q?=E5=9C=B0=E4=B8=8B=E9=9D=A2=E7=A7=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia-open/src/main/webapp/static/js/select.js | 11 ++++++++++- .../src/main/webapp/weiXinEvaluation/bhform.ftl | 7 +++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/webapp/static/js/select.js b/projects/fangjia-open/src/main/webapp/static/js/select.js index 6152b33bc..b7c01d2f6 100644 --- a/projects/fangjia-open/src/main/webapp/static/js/select.js +++ b/projects/fangjia-open/src/main/webapp/static/js/select.js @@ -152,7 +152,8 @@ $(document).on('click', '.create-actions03', function () { $("#buildType").val("8") $("#houseFloor").show() $("#floorDesc").hide() - $("#basementAreaLi").hide() + $("#basementAreaLi").show() + $("#basementAreaLi_intr").show() } }, { @@ -163,6 +164,7 @@ $(document).on('click', '.create-actions03', function () { $("#floorDesc").show() // $("#houseFloor").hide() $("#basementAreaLi").show() + $("#basementAreaLi_intr").show() } }, { @@ -173,6 +175,7 @@ $(document).on('click', '.create-actions03', function () { $("#floorDesc").show() // $("#houseFloor").hide() $("#basementAreaLi").show() + $("#basementAreaLi_intr").show() } }, { @@ -183,6 +186,7 @@ $(document).on('click', '.create-actions03', function () { $("#floorDesc").show() // $("#houseFloor").hide() $("#basementAreaLi").show() + $("#basementAreaLi_intr").show() } }, { @@ -193,6 +197,7 @@ $(document).on('click', '.create-actions03', function () { $("#floorDesc").show() // $("#houseFloor").hide() $("#basementAreaLi").show() + $("#basementAreaLi_intr").show() } }, { @@ -203,6 +208,7 @@ $(document).on('click', '.create-actions03', function () { $("#floorDesc").show() // $("#houseFloor").hide() $("#basementAreaLi").show() + $("#basementAreaLi_intr").show() } }, { @@ -213,6 +219,7 @@ $(document).on('click', '.create-actions03', function () { $("#floorDesc").show() // $("#houseFloor").hide() $("#basementAreaLi").show() + $("#basementAreaLi_intr").show() } }, { @@ -223,6 +230,7 @@ $(document).on('click', '.create-actions03', function () { $("#floorDesc").show() // $("#houseFloor").hide() $("#basementAreaLi").hide() + $("#basementAreaLi_intr").hide() } }, { @@ -233,6 +241,7 @@ $(document).on('click', '.create-actions03', function () { $("#houseFloor").show() // $("#floorDesc").hide() $("#basementAreaLi").hide() + $("#basementAreaLi_intr").hide() } } ]; diff --git a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl index 0604782cd..8a39228e1 100644 --- a/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl +++ b/projects/fangjia-open/src/main/webapp/weiXinEvaluation/bhform.ftl @@ -172,6 +172,13 @@
+
  • +
    +
    + *寤虹瓚闈㈢Н涓惈鏈夊湴涓嬮檮灞為潰绉� +
    +
    +
  • From 0df8ff3d21f2aad1c3033753907fcbdef17a0053 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Mar 2022 13:31:19 +0800 Subject: [PATCH 196/212] update --- .../ccbevaluation/CcbFtpTestUtils.groovy | 12 + .../fangjia/script/ccbevaluation/Test.groovy | 68 +++--- .../ronghe/BigdataCommunityGJRHBatch.groovy | 26 +-- .../ronghe/ExportDistrictRongHeTest.groovy | 5 + .../groovy/com/fangjia/script/gj/Test.groovy | 28 ++- .../script/gj/level/DsElasticUtils.groovy | 215 ++++++++++++++++++ .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 12 +- .../script/gj/price/FzMatchTest.groovy | 85 +++++++ .../script/thirdPriceCrawl/DataStat.groovy | 8 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 142 +++++++++++- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- .../TestCrawlTotalCount.groovy | 4 +- 13 files changed, 548 insertions(+), 65 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy index 4190d947b..7083cb615 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/CcbFtpTestUtils.groovy @@ -27,9 +27,21 @@ class CcbFtpTestUtils { }else{ sftp.upload(ftpPath+ month+"/", outputPath_dist, sftpChannel); } + } + + + static def delete(path){ + if(path.startsWith("/")){ + sftp.deleteSFTP(path, sftpChannel) + }else{ + sftp.deleteFile(path) + } } + + + static def reConnect(){ if(sftpChannel == null){ println "sftpChannel 为空,需重连....." diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy index 0b63403ce..6a9d4446e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/Test.groovy @@ -270,24 +270,24 @@ def youdaDeal(youda_esfhouse_coll, def city, def region, def name, def area, def // } //},0) -//def index = 0 -//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\320100_南京_是否独有及第三方是否存在(删除1049).xlsx",{d-> -// def flag = d['M'] -// if(flag==0.0){ -// if(d['D']){ -// def udata = new BasicDBObject() -// udata.put("name_cat", 11) -// udata.put("m_date",new Date()) -// -// def q = new BasicDBObject() -// q.put("_id", new ObjectId(d['D'])) -// DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) -// index++ -// println index -// } -// } -// -//},0) +def index = 0 +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\人工核实需删除数据\\是否独有及第三方是否存在20220217\\9370300_淄博_是否独有及第三方是否存在(删除383).xlsx",{d-> + def flag = d['M'] + if(flag==0.0){ + if(d['D']){ + def udata = new BasicDBObject() + udata.put("name_cat", 11) + udata.put("m_date",new Date()) + + def q = new BasicDBObject() + q.put("_id", new ObjectId(d['D'])) + DbUtils.seaweed.update(q, new BasicDBObject('$set', udata)) + index++ + println index + } + } + +},0) //ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\补充删除数据1223_1228.xlsx",{d-> // if(d['A']){ @@ -782,22 +782,22 @@ def getStreetInfo(def city, def glocation){ //} //ExcelUtils.write("C:\\Users\\gaojuan\\Desktop\\上海小区.xlsx", rets, "数据") -def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) -def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) - -def keys = [] -for(def d : datas2){ - keys << d['A']+"##"+d['C']+"##"+d['B'] -} - -def index = 0 -for(def d : datas1){ - def k = d['A']+"##"+d['C']+"##"+d['B'] - if(!keys.contains(k)){ - index ++ - println index +" "+k - } -} +//def datas1 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\12月.xlsx", 0, 1) +//def datas2 = ExcelUtils.read("C:\\Users\\gaojuan\\Desktop\\1月.xlsx", 0, 1) +// +//def keys = [] +//for(def d : datas2){ +// keys << d['A']+"##"+d['C']+"##"+d['B'] +//} +// +//def index = 0 +//for(def d : datas1){ +// def k = d['A']+"##"+d['C']+"##"+d['B'] +// if(!keys.contains(k)){ +// index ++ +// println index +" "+k +// } +//} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy index 6409c9795..dc70af53b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/BigdataCommunityGJRHBatch.groovy @@ -90,19 +90,19 @@ class BigdataCommunityGJRHBatch extends BaseBatch { dist.estatecharge = CcbCodeUtils.processEstateCharge(dist.city, obj.estate_charge, obj.estate_type2) } } -// println index +" "+city +" 小区加载完成 "+districts.size() -// addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 1.小区表完成" -// addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 2.小区建筑结构表" -// addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 3.小区房屋类型表" -// addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 4.小区建筑年代表" -// addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 5.小区费用表" -// addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) -// println index +" "+city +" 6.小区房屋用途表" + println index +" "+city +" 小区加载完成 "+districts.size() + addBigdataCommunityGJRH(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 1.小区表完成" + addBigdataCommbuildstruGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 2.小区建筑结构表" + addBigdataCommhousetypeGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 3.小区房屋类型表" + addBigdataCommbuildyearGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 4.小区建筑年代表" + addBigdataCommfeeGj(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 5.小区费用表" + addBigdataCommhouseuseGJ(districts, cityCodeDataMap.get(city)) + println index +" "+city +" 6.小区房屋用途表" addBigdataPositionGJ(districts, cityCodeDataMap.get(city), citySeqnoMap) println index +" "+ city + " 7.地理位置表" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy index 0c1c0c558..11270dab3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHeTest.groovy @@ -26,6 +26,11 @@ class ExportDistrictRongHeTest extends BaseBatch{ @Override void process(Object args) { def curDate = 20220310//DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") + def fileName = "/ccb_rh/${curDate}" +// if(CommonDatautils.isWindow()){ +// fileName = "D:\\数据\\建行\\房产估值\\数据文件\\txt\\ccb_rh\\${curDate}" +// } + CcbFtpTestUtils.delete(fileName) exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy index f2dbd6ff8..2996353c6 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/Test.groovy @@ -6,6 +6,7 @@ import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.HttpClient import com.fangjia.script.util.MongoUtils +import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.qqmap.TengXunApiUtils import com.mongodb.BasicDBObject import com.mongodb.Bytes @@ -42,4 +43,29 @@ println new HttpClient().get("https://longyuanguojiguangchangbl.fang.com/","gbk" // } // //} -//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据") \ No newline at end of file +//ExcelUtils.write("D:\\数据\\结果\\区域信息.xlsx", ret, "数据") + +ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆农商行测试地址.xlsx",{d-> + if(d['B']){ + def json = GaoDeApiUtils.geo("重庆", d['B']) + if(json && json.info=="OK"){ + def datas = json.geocodes + if(datas){ + def row = datas[0] + println row + d['C'] = row.country + d['D'] = row.province + d['E'] = row.city.toString().replace("[]","") + d['F'] = row.citycode.toString().replace("[]","") + d['G'] = row.district.toString().replace("[]","") + d['H'] = row.street.toString().replace("[]","") + d['I'] = row.number.toString().replace("[]","") + d['J'] = row.adcode.toString().replace("[]","") + d['K'] = row.location + d['L'] = row.level + } + } + } + + +}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy new file mode 100644 index 000000000..9ef7e6926 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DsElasticUtils.groovy @@ -0,0 +1,215 @@ +package com.fangjia.script.util + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.client.config.RequestConfig +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.bulk.BackoffPolicy +import org.elasticsearch.action.bulk.BulkProcessor +import org.elasticsearch.action.bulk.BulkRequest +import org.elasticsearch.action.bulk.BulkResponse +import org.elasticsearch.action.index.IndexRequest +import org.elasticsearch.action.search.ClearScrollRequest +import org.elasticsearch.action.search.SearchRequest +import org.elasticsearch.action.search.SearchResponse +import org.elasticsearch.action.search.SearchScrollRequest +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.search.Scroll +import org.elasticsearch.search.SearchHit +import org.elasticsearch.search.builder.SearchSourceBuilder +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.TimeUnit +import java.util.function.BiConsumer + +/** + * 功能说明:Elasticsearch工具类 + */ +class DsElasticUtils { + + + static getEsDataSource(String host){ + EsDataSource dataSource = new EsDataSource (); + dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240000&network.timeout=60000&page.timeout=240000&query.timeout=90000"); + Properties connectionProperties = new Properties(); + connectionProperties.setProperty("user","developer"); + connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); + dataSource.setProperties(connectionProperties); + return dataSource; + } + + + static RestHighLevelClient getClient(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 180); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) + } + + + /** + * 创建一个ES批处理器 + * @param host + * @return + */ + static createBulkProcessor(String host, RestHighLevelClient client = null) { + + BulkProcessor.Listener listener = new BulkProcessor.Listener() { + @Override + void beforeBulk(long executionId, BulkRequest request) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + BulkResponse response) { + } + + @Override + void afterBulk(long executionId, BulkRequest request, + Throwable failure) { + } + }; + if(!client){ + client = getClient(host); + } + return BulkProcessor.builder(new BiConsumer>() { + @Override + void accept(BulkRequest bulkRequest, ActionListener bulkResponseActionListener) { + client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener) + } + }, listener).setBulkActions(10000) + .setConcurrentRequests(20) + .setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)) + .setFlushInterval(TimeValue.timeValueSeconds(10L)).build(); + } + + /** + * 关闭一个ES批处理器 + * @param + * @return + */ + static closeBulkProcessor(BulkProcessor bulkProcessor) { + bulkProcessor.awaitClose(10, TimeUnit.DAYS) + bulkProcessor.flush() + bulkProcessor.close() + } + + + static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){ + + RestHighLevelClient client = getClient(host) + BulkProcessor bulkProcessor = createBulkProcessor(host, client) + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + def nThread = Runtime.getRuntime().availableProcessors() * 5 + ExecutorService executorService = Executors.newFixedThreadPool(nThread) + while (searchHits != null && searchHits.length > 0) { + for (int k=0; k < searchHits.size(); k++) { + final SearchHit hit = searchHits[k]; + executorService.submit(new Runnable() { + @Override + void run() { + try{ + if(closure.getMaximumNumberOfParameters() == 1){ + def update = closure.call(hit.getSourceAsMap()) + if(update){ + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } + }else if(closure.getMaximumNumberOfParameters() == 2){ + def update = closure.call(hit.getId(), hit.getSourceAsMap()) + if(update){ + bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap())) + } + } + }catch (Exception e){ + e.printStackTrace() + } + } + }) + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + executorService.shutdown() + executorService.awaitTermination(1, TimeUnit.DAYS) + bulkProcessor.close() + bulkProcessor.awaitClose(1, TimeUnit.DAYS) + client.close() + } + + + static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){ + + final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L)) + RestHighLevelClient client = getClient(host) + SearchRequest searchRequest = new SearchRequest(indexName); + searchRequest.scroll(scroll); + SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder(); + searchSourceBuilder.query(qb); + searchRequest.source(searchSourceBuilder); + SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT); + String scrollId = searchResponse.getScrollId(); + SearchHit[] searchHits = searchResponse.getHits().getHits(); + def results = [] + while (searchHits != null && searchHits.length > 0) { + for (int k=0; k < searchHits.size(); k++) { + SearchHit searchHit = searchHits[k] + searchHit.getSourceAsMap().put("id", searchHit.getId()); + results << searchHit.getSourceAsMap() + } + SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId); + scrollRequest.scroll(scroll); + searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT); + scrollId = searchResponse.getScrollId(); + searchHits = searchResponse.getHits().getHits(); + } + ClearScrollRequest clearScrollRequest = new ClearScrollRequest(); + clearScrollRequest.addScrollId(scrollId); + client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT); + client.close() + return results; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 320b85cdb..5f2bbe1b3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -110,7 +110,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -141,7 +141,7 @@ public class BeiKeCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index c6eec9d7e..76719fb79 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,10 +34,10 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() // task.addDealRegion(); - task.standardRegion() +// task.standardRegion() } def run1(){ @@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-11-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-30", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-31", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -419,9 +419,9 @@ public class LianjiaCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - // if(!href.contains("fang.lianjia") && e.text().contains("福州")){ +// if(!href.contains("fang.lianjia") && e.text().contains("福州")){ cityMap.put(e.text(), href) - // } +// } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy new file mode 100644 index 000000000..06218bbcf --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/price/FzMatchTest.groovy @@ -0,0 +1,85 @@ +package com.fangjia.script.gj.price + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess +import com.fangjia.script.util.ExcelUtils +import groovy.sql.Sql + +class FzMatchTest { + + def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", + "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + + static main(args) { + def task = new FzMatchTest(); +// task.run() + task.caluatePPE() + } + + def caluatePPE(){ + def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区_结果1).xlsx" + ExcelUtils.multiThreadingModify(filePath, { data -> + if(data['F'] && data['G']){ + def dist = CommonDatautils.findDistrict("福州", data['F'], data['G'], "district") + if(dist){ + def avg_price = findAvgPrice(dist._id.toString()) + if(data['i'] && avg_price){ + def weekendPrice = data['i'].toString().toDouble() + def ppe15 = Math.abs((weekendPrice - avg_price) * 100 / avg_price) + data['J'] = ppe15 + } + } + } + },1) + } + + def findAvgPrice(def commid){ + def sql = "select year_month,target_avg_price from (select year_month,target_avg_price from avm_evaluation_item_gj where commid='${commid}' order by year_month desc) where rownum<2"; + def avg_price = null + sqlInstance.eachRow(sql, { r -> + avg_price = r.target_avg_price + }); + return avg_price + } + + def run(){ + def filePath = "D:\\数据\\福州\\商品住房楼盘评估指导参考价情况表(估家科技_福州五区).xlsx" + def city = "福州" + ExcelUtils.multiThreadingModify(filePath, { data -> +// if(data['G']){ +// if(!data['H']){ +// def dist = CommonDatautils.findDistrict(city, data['F'], data['G'], "district") +// +// if(dist && dist.finish_building_date){ +// data['H'] = dist.finish_building_date.substring(0,4) +// } +// } +// if(!data['I']){ +// def price = CommonDatautils.queryDistrictPrice(city, data['F'], data['G']) +// if(price){ +// data['I'] = price +// } +// } +// +// } + + if(!data['G'] && data['B']){ + def name = data['B'].replace("(", "").replace(")","").replace("\\","").replace("#","") + def rMap = SeaweedDataMatchProcess.doMatch(city, null, name, null); + if(rMap) { + data['F'] = rMap.region + data['G'] = rMap.name + def dist = CommonDatautils.findDistrictById(rMap.id) + if(dist && dist.finish_building_date){ + data['H'] = dist.finish_building_date.substring(0,4) + } + + def price = CommonDatautils.queryDistrictPrice(city, rMap.region, rMap.name) + if(price){ + data['I'] = price + } + } + } + }, 1); + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 7f04edab1..5443a7a6f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,11 +40,11 @@ class DataStat { def run(){ - def source = "城市房产"// - def citys = findCrawlCity(source, 202112); + def source = "58同城"// + def citys = findCrawlCity(source, 202201); for(def city : citys){ - def count1 = findCount(source, city, 202112); - def count2 = findCount(source, city, 202201); + def count1 = findCount(source, city, 202201); + def count2 = findCount(source, city, 202202); println city +" "+count1+" "+count2 // println count1 // println "--- ----" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 5fb2fef63..69aaa9179 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -181,7 +181,147 @@ class Esf58Crawl extends BaseBatch{ // "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", // "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" def cityList = [ - "福州":"fz|304" + "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", + "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", + "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", + "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", + "呼和浩特":"hu|811", + "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", + "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", + "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", + "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", + "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", + "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", + "淮北":"huaibei|9357","滁州":"chuzhou|10266", + "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", + "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", + "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", + "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", + "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", + "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", + "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", + "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", + "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", + "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", + "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", + "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", + "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", + "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", + "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", + "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", + "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", + "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", + "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", + "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", + "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", + "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", + "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", + "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", + "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", + "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", + "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", + "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", + "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", + "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", + "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", + "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", + "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", + "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", + "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", + "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", + "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", + "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", + "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", + "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", + "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", + "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", + "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", + "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", + "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", + "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", + "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", + "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", + "邢台":"xt|751","衡水":"hs|993", + "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", + "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", + "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", + "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", + "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", + "沙河":"shaheshi|755", + "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", + "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", + "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", + "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", + "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", + "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", + "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", + "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", + "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", + "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", + "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", + "浮梁":"fuliangxian|9071", + "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", + "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", + "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", + "梨树县":"lishu|10176", + "鞍山":"as|523","抚顺":"fushun|5722", + "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", + "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", + "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", + "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", + "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", + "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", + "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", + "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", + "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", + "泰安":"ta|686","聊城":"lc|882", + "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", + "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", + "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", + "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", + "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", + "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", + "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", + "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", + "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", + "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", + "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", + "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", + "临朐":"linqu|374","鄄城":"juancheng|5635", + "运城":"yuncheng|5653","晋中":"jz|8854", + "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", + "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", + "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", + "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", + "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", + "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", + "德阳":"deyang|2373","宜宾":"yb|2380", + "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", + "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", + "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", + "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", + "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", + "昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", + "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", + "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", + "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", + "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", + "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", + "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", + "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", + "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", + "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", + "丽水":"lishui|7921","衢州":"quzhou|6793", + "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", + "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", + "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", + "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", + "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", + "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" ] static main(args) { diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 0b75f8545..2d055dcbb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -15,10 +15,10 @@ class SynchDistrictPrice { def static final SHORT_DATE = Utils.getLastMonth() def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def districtDetailColl = CrawalDBUtil.getDistrictDetailColl(); -// def sourceMap = ["jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fang.com":"搜房","fangdd.com":"房多多","house365.com":"365淘房", +// def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] //"fang.com":"搜房", - def sourceMap = ["ke.com":"贝壳找房","cityhouse.cn":"城市房产"] + def sourceMap = ["cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy index ed17686b9..db48455df 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/TestCrawlTotalCount.groovy @@ -14,8 +14,8 @@ class TestCrawlTotalCount { def run(){ def sources = ["房估估","诸葛找房","安居客","搜房","中原地产","赶集","链家","兔博士","城市房产","贝壳找房","Q房","Q房(新)","中原地产(新)","乐有家","乐有家(新)","房探","房多多","小资家","我爱我家","麦田","房途","C21","乐居","房产超市","58同城","365淘房","吉屋"] for(def source :sources){ - def count1 = findTotalCount(source, 202112); - def count2 = findTotalCount(source, 202201); + def count1 = findTotalCount(source, 202201); + def count2 = findTotalCount(source, 202202); println source +" "+count1 +" "+count2 } } From fbef302f79dfe7735a7ff88abad978a9e3ec58ec Mon Sep 17 00:00:00 2001 From: gaojuan Date: Wed, 9 Mar 2022 13:31:39 +0800 Subject: [PATCH 197/212] update --- .../fangjia/script/jiangnanblank/SFTP.groovy | 51 ++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy index 811de10a1..70082d882 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/SFTP.groovy @@ -42,6 +42,55 @@ class SFTP { return sftp; } + + + /** + * 删除本地文件 + * + * @param filePath + * @return + */ + public boolean deleteFile(String filePath) { + File file = new File(filePath); + if (!file.exists()) { + return false; + } + if (!file.isFile()) { + file.deleteDir(); + return false; + } + boolean rs = file.delete(); + return rs; + } + + public static void deleteSFTP(String directory, ChannelSftp sftp) { + try { + if (isDirExist(directory, sftp)) { + Vector vector = sftp.ls(directory); + if (vector.size() == 1) { // 文件,直接删除 + sftp.rm(directory); + } else if (vector.size() == 2) { // 空文件夹,直接删除 + sftp.rmdir(directory); + } else { + String fileName = ""; + // 删除文件夹下所有文件 + for (ChannelSftp.LsEntry en : vector) { + fileName = en.getFilename(); + if (".".equals(fileName) || "..".equals(fileName)) { + continue; + } else { + deleteSFTP(directory + "/" + fileName); + } + } + // 删除文件夹 + sftp.rmdir(directory); + } + } + } catch (Exception e) { + e.printStackTrace(); + } + } + /** * 上传文件 * @param directory 上传的目录 @@ -132,7 +181,7 @@ class SFTP { sftp.cd(filePath.toString()); } else { // 建立目录 - sftp.mkdir(filePath.toString()); + sftp.mkdir(filePath.toString()) // 进入并设置为当前目录 sftp.cd(filePath.toString()); } From fbd6971e232b968b054e3dd8c1fb5769d8039e26 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 10 Mar 2022 10:35:00 +0800 Subject: [PATCH 198/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E6=A0=87=E5=87=86=E5=8C=96=E6=B5=8B=E8=AF=95=E4=B8=93=E7=94=A8?= =?UTF-8?q?=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index d11473fcc..746bdaa91 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1814,7 +1814,7 @@ public class AddressController { HttpRequest request = new HttpRequest(); JSONObject json = null; Map params = new HashMap(); - params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : "重庆市"); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); params.put("address", model.getAddress()); // 地址验证无效也输出标准化结果 params.put("flag", "true"); @@ -1842,6 +1842,50 @@ public class AddressController { return mav; } + /** + * 地址标准化专用测试接口 + * @param model + * @return + */ + public Object structureForTestAction(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureForTest", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureForTest", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + public Object similarityAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress_1()) || StringUtils.isBlank(model.getAddress_2())) { From 91560b90c13da880c226b410c8b1dd289caf5592 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 10 Mar 2022 16:32:56 +0800 Subject: [PATCH 199/212] =?UTF-8?q?=E5=A1=AB=E5=85=85=E5=9F=8E=E4=B9=A1?= =?UTF-8?q?=E5=88=86=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/address/FillAgriculture.groovy | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy new file mode 100644 index 000000000..99acbc19c --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy @@ -0,0 +1,56 @@ +package com.fangjia.script.temp.shenjian.address + +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.fangjia.script.util.ProvinceData +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.apache.commons.lang3.StringUtils + +final DBCollection townConn = MongoUtils.getCollectionByName("fangjia_crawl", "regionalism_code", "192.168.0.235", 27018) + +ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识库\\重庆农商行\\重庆农商行测试地址(估家科技)V2.xlsx", { + List item -> + def id = item['A'] + def province = item['D'] + def city = item['E'] + def area = item['F'] + def township = item['G'] + def community = item['H'] + if (StringUtils.isBlank(city)) { + return ; + } + if (ProvinceData.isGovernmentCity(city)) { + townConn.find(new BasicDBObject("province", "重庆市") + .append("region", area + "区") + .append("street", township) + .append("juweihui", community), new BasicDBObject()).limit(1).each { + obj -> + item['T'] = obj.juweihui_type + } + if (item['T'] == null) { + townConn.find(new BasicDBObject("province", "重庆市") + .append("region", area + "区") + .append("juweihui", community), new BasicDBObject()).limit(1).each { + obj -> + item['T'] = obj.juweihui_type + } + } + } else { + townConn.find(new BasicDBObject("city", city + "市") + .append("region", area + "区") + .append("street", township) + .append("juweihui", community), new BasicDBObject()).limit(1).each { + obj -> + item['T'] = obj.juweihui_type + } + if (item['T'] == null) { + townConn.find(new BasicDBObject("city", city + "市") + .append("region", area + "区") + .append("juweihui", community), new BasicDBObject()).limit(1).each { + obj -> + item['T'] = obj.juweihui_type + } + } + } +}) From 322f7ab0413c9adfb28674251e3d7bc326b983bd Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Thu, 10 Mar 2022 17:59:48 +0800 Subject: [PATCH 200/212] =?UTF-8?q?=E5=A1=AB=E5=85=85=E5=9F=8E=E4=B9=A1?= =?UTF-8?q?=E5=88=86=E7=B1=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../shenjian/address/FillAgriculture.groovy | 86 +++++++++++-------- 1 file changed, 52 insertions(+), 34 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy index 99acbc19c..7993a0b8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/address/FillAgriculture.groovy @@ -15,42 +15,60 @@ ExcelUtils.multiThreadingModify("D:\\Users\\Domi\\Documents\\知识 def province = item['D'] def city = item['E'] def area = item['F'] - def township = item['G'] - def community = item['H'] + String township = item['G'] + String community = item['H'] if (StringUtils.isBlank(city)) { return ; } - if (ProvinceData.isGovernmentCity(city)) { - townConn.find(new BasicDBObject("province", "重庆市") - .append("region", area + "区") - .append("street", township) - .append("juweihui", community), new BasicDBObject()).limit(1).each { - obj -> - item['T'] = obj.juweihui_type - } - if (item['T'] == null) { - townConn.find(new BasicDBObject("province", "重庆市") - .append("region", area + "区") - .append("juweihui", community), new BasicDBObject()).limit(1).each { - obj -> - item['T'] = obj.juweihui_type - } - } - } else { - townConn.find(new BasicDBObject("city", city + "市") - .append("region", area + "区") - .append("street", township) - .append("juweihui", community), new BasicDBObject()).limit(1).each { - obj -> - item['T'] = obj.juweihui_type - } - if (item['T'] == null) { - townConn.find(new BasicDBObject("city", city + "市") - .append("region", area + "区") - .append("juweihui", community), new BasicDBObject()).limit(1).each { - obj -> - item['T'] = obj.juweihui_type - } - } + if (item['T'] != null) { + return ; } +// if (community == null) { +// return ; +// } + if (township) { + if (township.endsWith("镇") || township.endsWith("乡")) { + item['T'] = 2 + } else if (township.endsWith("街道")) { + item['T'] = 1 + } + } +// community = community.replaceAll("居委会", "") +// community = community.replaceAll("居民委员会", "") +// community = community.replaceAll("村民委员会", "") +// community = community.replaceAll("村委会", "") +// +// if (ProvinceData.isGovernmentCity(city)) { +// townConn.find(new BasicDBObject("province", "重庆市") +// .append("region", area + "区") +// .append("street", township) +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// if (item['T'] == null) { +// townConn.find(new BasicDBObject("province", "重庆市") +// .append("region", area + "区") +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// } +// } else { +// townConn.find(new BasicDBObject("city", city + "市") +// .append("region", area + "区") +// .append("street", township) +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// if (item['T'] == null) { +// townConn.find(new BasicDBObject("city", city + "市") +// .append("region", area + "区") +// .append("juweihui", new BasicDBObject('$regex', "^" + community)), new BasicDBObject()).limit(1).each { +// obj -> +// item['T'] = obj.juweihui_type +// } +// } +// } }) From 1b71354b53f4bdd5d49938d39a6a694cdc03c878 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 11 Mar 2022 09:37:43 +0800 Subject: [PATCH 201/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E5=9C=B0=E5=9D=80?= =?UTF-8?q?=E8=A1=A5=E5=85=A8=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/open/controllers/AddressController.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 746bdaa91..2fa5a4b4c 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1847,7 +1847,7 @@ public class AddressController { * @param model * @return */ - public Object structureForTestAction(AddesssMatchModel model) { + public Object structureAndCompleteAction(AddesssMatchModel model) { RestfulModelAndView mav = new RestfulModelAndView(); if (StringUtils.isBlank(model.getAddress())) { @@ -1863,10 +1863,10 @@ public class AddressController { // 地址验证无效也输出标准化结果 params.put("flag", "true"); try{ - json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureForTest", params, "UTF-8"); + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8"); } catch(Exception e){ try { - json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureForTest", params, "UTF-8"); + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/structureAndComplete", params, "UTF-8"); } catch(Exception ex){ ex.printStackTrace(); } From 301bcb09844eedfb3f87a647872e36d2f1bb2470 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Mar 2022 15:32:03 +0800 Subject: [PATCH 202/212] =?UTF-8?q?=E5=B0=8F=E5=8C=BA=E6=BD=9C=E5=8A=9B?= =?UTF-8?q?=E6=8C=87=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 1 + .../script/fuzhou/FzDataResultStat.groovy | 2 +- .../script/gj/level/DistrictIndexBatch.groovy | 317 +++++++++++-- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 449 +++++++----------- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 5 files changed, 441 insertions(+), 330 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index a3e425055..dd91fbeaf 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -830,6 +830,7 @@ class CommonDatautils { keys.put("lng", 1); keys.put("name_cat", 1); keys.put("finish_building_date", 1) + keys.put("mapping_alias", 1) DSCursor cursor = getDataStoreConnection().find(q, keys); if(cursor){ for(def obj : cursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy index 8b5fd5715..ba848d886 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/FzDataResultStat.groovy @@ -20,7 +20,7 @@ class FzDataResultStat { static main(args) { def task = new FzDataResultStat() task.priceMarkupRun(); -// task.regionPriceTrendRun() + task.regionPriceTrendRun() } def priceMarkupRun(){ def curMonth = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 5547c7ce9..8feaa621c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -1,15 +1,18 @@ package com.fangjia.script.gj.level +import com.fangjia.core.util.DataUtil import com.fangjia.core.util.HttpRequest import com.fangjia.httpclient.UserAgents import com.fangjia.script.BaseBatch import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.EsUtils import com.fangjia.script.gj.ccb.FileUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.JacksonUtil @@ -18,7 +21,11 @@ import groovy.sql.Sql import net.sf.json.JSONObject import org.apache.commons.httpclient.util.DateUtil import org.apache.commons.lang.StringUtils +import org.apache.commons.math.stat.descriptive.moment.StandardDeviation +import org.apache.commons.math3.stat.descriptive.moment.Mean import org.bson.types.ObjectId +import org.elasticsearch.index.query.QueryBuilder +import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet @@ -44,9 +51,9 @@ class DistrictIndexBatch extends BaseBatch{ dataSource.setProperties(connectionProperties) } - static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/_doc" + static def targetUrl = "http://192.168.0.12:9200/district_comp_potential/" - public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(40) + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(30) def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") @@ -78,7 +85,9 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + calcute(city, dists, code, city_avg_total_price) + calcuteIndex(city, dists, code, city_avg_total_price) + } println city +" "+city_avg_total_price } @@ -110,46 +119,25 @@ class DistrictIndexBatch extends BaseBatch{ def total_price = getHousePrice(d, dists, priceMap, citycode); def avg_cost = getAvgCost(d, costDatas) - def total_price_index = 0 - if (total_price) { - total_price_index = 286.0134196 * (total_price - city_avg_total_price) / 313.3693508 - total_price_index = DataHandleUtils.getDoubleRound(6, total_price_index) - } - def avg_charge_index = 0 - if (avg_charge) { - avg_charge_index = 139.1374524 * (avg_charge - 1.475882691) / 0.772635241 - avg_charge_index = DataHandleUtils.getDoubleRound(6, avg_charge_index) - } - def rank_index = 0 - if (rank) { - rank_index = 21.8427926 * (rank - 0.591827186) / 0.261907636 - rank_index = DataHandleUtils.getDoubleRound(6, rank_index) - } - - def avg_cost_index = 0 - if (avg_cost) { - avg_cost_index = 67.0195871700 * (avg_cost - 73.04713255) / 31.3526959 - avg_cost_index = DataHandleUtils.getDoubleRound(6, avg_cost_index) - } - def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248 - potentiality = DataHandleUtils.getDoubleRound(6, potentiality) def map = [:] map.id = d._id.toString() map.city = d.city map.region = d.region map.name = d.name - map.avg_total_price = total_price_index + map.avg_estate_charge = avg_charge map.price_rank = rank map.avg_cost = avg_cost - map.comp_value = potentiality - map.month = "2022-01" + map.total_price = total_price + //map.comp_value = potentiality + //map.avg_total_price = total_price_index + map.month = "2022-03" map.c_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") - datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] + //datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] - def newUrl = targetUrl +"/"+map.id + def newUrl = targetUrl +"_doc/"+map.id httpSaveData(newUrl, map) println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price } catch (Exception e) { @@ -163,16 +151,266 @@ class DistrictIndexBatch extends BaseBatch{ latch.await(); - def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] - def fileName = "/home/gj/potentiality_${city}.dat" - if(CommonDatautils.isWindow()){ - fileName = "D:\\数据\\评级\\${city}综合潜力.dat" +// def headers = ["城市","区域","名称","户均价格","物业费","小区价格在区域排名","消费","潜力"] +// def fileName = "/home/gj/potentiality_${city}.dat" +// if(CommonDatautils.isWindow()){ +// fileName = "D:\\数据\\评级\\${city}综合潜力.dat" +// } +// FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); +// // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") + } + + def calcuteIndex(city, dists, citycode, city_avg_total_price){ + def detailMap = findAllDistrictDatas(city) + def avgMap = calcuteAvgValues(detailMap); + def totalCount = new AtomicLong() + final CountDownLatch latch = new CountDownLatch(dists.size()) + + def avg_total_price = avgMap.avg_price + def avg_charge = avgMap.avg_charge + def avg_cost = avgMap.avg_cost + def avg_rank = avgMap.avg_rank + + def sd_avg_price = avgMap.sd_avg_price + def sd_avg_charge = avgMap.sd_avg_charge + def sd_avg_cost = avgMap.sd_avg_cost + def sd_avg_rank = avgMap.sd_avg_rank + + for(def i=0;i + if(v.total_price){ + prices << v.total_price + } + if(v.price_rank){ + ranks << v.price_rank + } + if(v.avg_cost){ + costs << v.avg_cost + } + if(v.avg_estate_charge){ + charges << v.avg_estate_charge + } } - FileUtils.batchCreateFile(datas, fileName, headers, "utf-8", false); - // ExcelUtils.write("D:\\数据\\评级\\上海综合潜力.xlsx", datas, "数据") + def avg_price = null + def dx_avg_price = null + def sd_avg_price = null + if(prices){ + def v = prices.sum()/prices.size() + avg_price = DataHandleUtils.getDoubleRound(6, v) + + double pow_sum = 0 + prices.each {p-> + def sod = Math.abs(p - avg_price) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_price = DataHandleUtils.getDoubleRound(6,pow_sum/prices.size()) + sd_avg_price = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_price)) + } + def avg_rank = null + def dx_avg_rank = null + def sd_avg_rank = null + if(ranks){ + def v = ranks.sum()/ranks.size() + avg_rank = DataHandleUtils.getDoubleRound(6, v) + + def pow_sum = 0; + ranks.each {p-> + def sod = Math.abs(p - avg_rank) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_rank = DataHandleUtils.getDoubleRound(6,pow_sum/ranks.size()) + sd_avg_rank = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_rank)) + } + + def avg_cost = null + def dx_avg_cost = null + def sd_avg_cost = null + if(costs){ + def v = costs.sum()/costs.size() + avg_cost = DataHandleUtils.getDoubleRound(6, v) + + double pow_sum = 0; + costs.each {p-> + def sod = Math.abs(p - avg_cost) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_cost = DataHandleUtils.getDoubleRound(6,pow_sum/costs.size()) + sd_avg_cost = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_cost)) + } + + + def avg_charge = null + def dx_avg_charge = null + def sd_avg_charge = null + + if(charges){ + def v = charges.sum()/charges.size() + avg_charge = DataHandleUtils.getDoubleRound(6, v) + + def pow_sum = 0; + charges.each {p-> + def sod = Math.abs(p - avg_charge) + pow_sum = Math.pow(sod, 2) + pow_sum + } + dx_avg_charge = DataHandleUtils.getDoubleRound(6,pow_sum/charges.size()) + sd_avg_charge = DataHandleUtils.getDoubleRound(6, Math.sqrt(dx_avg_charge)) + } + + def avgMap = [:] + avgMap.put("avg_rank", avg_rank) + avgMap.put("avg_price", avg_price) + avgMap.put("avg_cost", avg_cost) + avgMap.put("avg_charge", avg_charge) + + avgMap.put("dx_avg_rank", dx_avg_rank) + avgMap.put("dx_avg_price", dx_avg_price) + avgMap.put("dx_avg_cost", dx_avg_cost) + avgMap.put("dx_avg_charge", dx_avg_charge) + + avgMap.put("sd_avg_rank", sd_avg_rank) + avgMap.put("sd_avg_price", sd_avg_price) + avgMap.put("sd_avg_cost", sd_avg_cost) + avgMap.put("sd_avg_charge", sd_avg_charge) + return avgMap; } + //方差s^2=[(x1-x)^2 +...(xn-x)^2]/n 或者s^2=[(x1-x)^2 +...(xn-x)^2]/(n-1) + public double Variance(def x) { + int m=x.size(); + double sum=0; + for(int i=0;i 0) { for (String head : headers.keySet()) { @@ -547,7 +786,7 @@ class DistrictIndexBatch extends BaseBatch{ } catch (Exception e) { e.printStackTrace() } - return null; + return buffer.toString(); } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 69aaa9179..37d2cafbb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -39,289 +39,149 @@ class Esf58Crawl extends BaseBatch{ def static useDistKeys = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","不","了","这","中","大","有","和","人","是","在","一","的","为","上","个","国","我","以","要","他","时","来","用","们","生","到","作","地","于","出","就","分","对","成","会","可","主","发","年","动","同","工","也","能","下","过","子","说","产","种","面","而","方","后","多","定","行","学","法","所","民","得","经","十","三","之","进","着","等","部","度","家","电","力","里","如","水","化","高","自","二","理","起","小","物","现","实","加","量","都","两","体","制","机","当","使","点","从","业","本","去","把","性","好","应","开","它","合","还","因","由","其","些","然","前","外","天","政","四","日","那","社","义","事","平","形","相","全","表","间","样","与","关","各","重","新","线","内","数","正","心","反","你","明","看","原","又","么","利","比","或","但","质","气","第","向","道","命","此","变","条","只","没","结","解","问","意","建","月","公","无","系","军","很","情","者","最","立","代","想","已","通","并","提","直","题","党","程","展","五","果","料","象","员","革","位","入","常","文","总","次","品","式","活","设","及","管","特","件","长","求","老","头","基","资","边","流","路","级","少","图","山","统","接","知","较","长","将","组","见","计","别","她","手","角","期","根","论","运","农","指","几","九","区","强","放","决","西","被","干","做","必","战","先","回","则","任","取","据","处","队","南","给","色","光","门","即","保","治","北","造","百","规","热","领","七","海","地","口","东","导","器","压","志","世","金","增","争","济","阶","油","思","术","极","交","受","联","什","认","六共","权","收","证","改","清","已","美","再","采","转","更","单","风","切","打","白","教","速","花","带","安","场","身","车","例","真","务","具","万","每","目","至","达","走","积","示","议","声","报","斗","完","类","八","离","华","名","确","才","科","张","信","马","节","话","米","整","空","元","况","今","集","温","传","土","许","步","群","广","石","记","需","段","研","界","拉","林","律","叫","且","究","观","越","织","装","影","算","低","持","音","众","书","布","复","容","儿","须","际","商","非","验","连","断","深","难","近","矿","千","周","委","素","技","备","半","办","青","省","列","习","响","约","支","般","史","感","劳","便","团","往","酸","历","市","克","何","除","消","构","府","称","太","准","精","值","号","率","族","维","划","选","标","写","存","候","毛","亲","快","效","斯","院","查","江","型","眼","王","按","格","养","易","置","派","层","片","始","却","专","状","育","厂","京","识","适","属","圆","包","火","住","调","满","县","局","照","参","红","细","引","听","该","铁","价","严","首","底","液","官","德","调","随","病","苏","失","尔","死","讲","配","女","黄","推","显","谈","罪","神","艺","呢","席","含","企","望","密","批","营","项","防","举","球","英","氧","势","告","李","台","落","木","帮","轮","破","亚","师","围","注","远","字","材","排","供","河","态","封","另","施","减","树","溶","怎","止","案","言","士","均","武","固","叶","鱼","波","视","仅","费","紧","爱","左","章","早","朝","害","续","轻","服","试","食","充","兵","源","判","护","司","足","某","练","差","致","板","田","降","黑","犯","负","击","范","继","兴","似","余","坚","曲","输","修","的","故","城","夫","够","送","笔","船","占","右","财","吃","富","春","职","觉","汉","画","功","巴","跟","虽","杂","飞","检","吸","助","升","阳","互","初","创","抗","考","投","坏","策","古","径","换","未","跑","留","钢","曾","端","责","站","简","述","钱","副","尽","帝","射","草","冲","承","独","令","限","阿","宣","环","双","请","超","微","让","控","州","良","轴","找","否","纪","益","依","优","顶","础","载","倒","房","突","坐","粉","敌","略","客","袁","冷","胜","绝","析","块","剂","测","丝","协","重","诉","念","陈","仍","罗","盐","友","洋","错","苦","夜","刑","移","频","逐","靠","混","母","短","皮","终","聚","汽","村","云","哪","既","距","卫","停","烈","央","察","烧","行","迅","境","若","印","洲","刻","括","激","孔","搞","甚","室","待","核","校","散","侵","吧","甲","游","久","菜","味","旧","模","湖","货","损","预","阻","毫","普","稳","乙","妈","植","息","扩","银","语","挥","酒","守","拿","序","纸","医","缺","雨","吗","针","刘","啊","急","唱","误","训","愿","审","附","获","茶","鲜","粮","斤","孩","脱","硫","肥","善","龙","演","父","渐","血","欢","械","掌","歌","沙","著","刚","攻","谓","盾","讨","晚","粒","乱","燃","矛","乎","杀","药","宁","鲁","贵","钟","煤","读","班","伯","香","介","迫","句","丰","培","握","兰","担","弦","蛋","沉","假","穿","执","答","乐","谁","顺","烟","缩","征","脸","喜","松","脚","困","异","免","背","星","福","买","染","井","概","慢","怕","磁","倍","祖","皇","促","静","补","评","翻","肉","践","尼","衣","宽","扬","棉","希","伤","操","垂","秋","宜","氢","套","笔","督","振","架","亮","末","宪","庆","编","牛","触","映","雷","销","诗","座","居","抓","裂","胞","呼","娘","景","威","绿","晶","厚","盟","衡","鸡","孙","延","危","胶","还","屋","乡","临","陆","顾","掉","呀","灯","岁","措","束","耐","剧","玉","赵","跳","哥","季","课","凯","胡","额","款","绍","卷","齐","伟","蒸","殖","永","宗","苗","川","炉","岩","弱","零","杨","奏","沿","露","杆","探","滑","镇","饭","浓","航","怀","赶","库","夺","伊","灵","税","了","途","灭","赛","归","召","鼓","播","盘","裁","险","康","唯","录","菌","纯","借","糖","盖","横","符","私","努","堂","域","枪","润","幅","哈","竟","熟","虫","泽","脑","壤","碳","欧","遍","侧","寨","敢","彻","虑","斜","薄","庭","都","纳","弹","饲","伸","折","麦","湿","暗","荷","瓦","塞","床","筑","恶","户","访","塔","奇","透","梁","刀","旋","迹","卡","氯","遇","份","毒","泥","退","洗","摆","灰","彩","卖","耗","夏","择","忙","铜","献","硬","予","繁","圈","雪","函","亦","抽","篇","阵","阴","丁","尺","追","堆","雄","迎","泛","爸","楼","避","谋","吨","野","猪","旗","累","偏","典","馆","索","秦","脂","潮","爷","豆","忽","托","惊","塑","遗","愈","朱","替","纤","粗","倾","尚","痛","楚","谢","奋","购","磨","君","池","旁","碎","骨","监","捕","弟","暴","割","贯","殊","释","词","亡","壁","顿","宝","午","尘","闻","揭","炮","残","冬","桥","妇","警","综","招","吴","付","浮","遭","徐","您","摇","谷","赞","箱","隔","订","男","吹","乐","园","纷","唐","败","宋","玻","巨","耕","坦","荣","闭","湾","键","凡","驻","锅","救","恩","剥","凝","碱","齿","截","炼","麻","纺","禁","废","盛","版","缓","净","睛","昌","婚","涉","筒","嘴","插","岸","朗","庄","街","藏","姑","贸","腐","奴","啦","惯","乘","伙","恢","匀","纱","扎","辩","耳","彪","臣","亿","璃","抵","脉","秀","萨","俄","网","舞","店","喷","纵","寸","汗","挂","洪","着","贺","闪","柬","爆","烯","津","稻","墙","软","勇","像","滚","厘","蒙","芳","肯","坡","柱","荡","腿","仪","旅","尾","轧","冰","贡","登","黎","削","钻","勒","逃","障","氨","郭","峰","币","港","伏","轨","亩","毕","擦","莫","刺","浪","秘","援","株","健","售","股","岛","甘","泡","睡","童","铸","汤","阀","休","汇","舍","牧","绕","炸","哲","磷","绩","朋","淡","尖","启","陷","柴","呈","徒","颜","泪","稍","忘","泵","蓝","拖","洞","授","镜","辛","壮","锋","贫","虚","弯","摩","泰","幼","廷","尊","窗","纲","弄","隶","疑","氏","宫","姐","震","瑞","怪","尤","琴","循","描","膜","违","夹","腰","缘","珠","穷","森","枝","竹","沟","催","绳","忆","邦","剩","幸","浆","栏","拥","牙","贮","礼","滤","钠","纹","弹","罢","拍","咱","喊","袖","埃","勤","罚","焦","潜","伍","墨","欲","缝","姓","刊","饱","仿","奖","铝","鬼","丽","跨","默","挖","链","扫","喝","袋","炭","污","幕","诸","弧","励","梅","奶","洁","灾","舟","鉴","苯","讼","抱","毁","率","懂","寒","智","埔","寄","届","跃","渡","挑","丹","艰","贝","碰","拔","爹","戴","码","梦","芽","熔","赤","渔","哭","敬","颗","奔","藏","铅","熟","仲","虎","稀","妹","乏","珍","申","桌","遵","允","隆","螺","仓","魏","锐","晓","氮","兼","隐","碍","赫","拨","忠","肃","缸","牵","抢","博","巧","壳","兄","杜","讯","诚","碧","祥","柯","页","巡","矩","悲","灌","龄","伦","票","寻","桂","铺","圣","恐","恰","郑","趣","抬","荒","腾","贴","柔","滴","猛","阔","辆","妻","填","撤","储","签","闹","扰","紫","砂","递","戏","吊","陶","伐","喂","疗","瓶","婆","抚","臂","摸","忍","虾","蜡","邻","胸","巩","挤","偶","弃","槽","劲","乳","邓","吉","仁","烂","砖","租","乌","舰","伴","瓜","浅","丙","暂","燥","橡","柳","迷","暖","牌","纤","秧","胆","详","簧","踏","瓷","谱","呆","宾","糊","洛","辉","愤","竞","隙","怒","粘","乃","绪","肩","籍","敏","涂","熙","皆","侦","悬","掘","享","纠","醒","狂","锁","淀","恨","牲","霸","爬","赏","逆","玩","陵","祝","秒","浙","貌","役","彼","悉","鸭","着","趋","凤","晨","畜","辈","秩","卵","署","梯","炎","滩","棋","驱","筛","峡","冒","啥","寿","译","浸","泉","帽","迟","硅","疆","贷","漏","稿","冠","嫩","胁","芯","牢","叛","蚀","奥","鸣","岭","羊","凭","串","塘","绘","酵","融","盆","锡","庙","筹","冻","辅","摄","袭","筋","拒","僚","旱","钾","鸟","漆","沈","眉","疏","添","棒","穗","硝","韩","逼","扭","侨","凉","挺","碗","栽","炒","杯","患","馏","劝","豪","辽","勃","鸿","旦","吏","拜","狗","埋","辊","掩","饮","搬","骂","辞","勾","扣","估","蒋","绒","雾","丈","朵","姆","拟","宇","辑","陕","雕","偿","蓄","崇","剪","倡","厅","咬","驶","薯","刷","斥","番","赋","奉","佛","浇","漫","曼","扇","钙","桃","扶","仔","返","俗","亏","腔","鞋","棱","覆","框","悄","叔","撞","骗","勘","旺","沸","孤","粘","吐","孟","渠","屈","疾","妙","惜","仰","狠","胀","谐","抛","霉","桑","岗","嘛","衰","盗","渗","脏","赖","涌","甜","曹","阅","肌","哩","厉","烃","纬","毅","昨","伪","症","煮","叹","钉","搭","茎","笼","酷","偷","弓","锥","恒","杰","坑","鼻","翼","纶","叙","狱","逮","罐","络","棚","抑","膨","蔬","寺","骤","穆","冶","枯","册","尸","凸","绅","坯","牺","焰","轰","欣","晋","瘦","御","锭","锦","丧","旬","锻","垄","搜","佛","扑","邀","亭","酯","迈","舒","脆","酶","闲","忧","酚","顽","羽","涨","卸","仗","陪","薄","辟","惩","杭","姚","肚","捉","飘","漂","昆","欺","吾","郎","烷","汁","呵","饰","萧","雅","邮","迁","燕","撒","姻","赴","宴","烦","削","债","帐","斑","铃","旨","醇","董","饼","雏","姿","拌","傅","腹","妥","揉","贤","拆","歪","葡","胺","丢","浩","徽","昂","垫","挡","览","贪","慰","缴","汪","慌","冯","诺","姜","谊","凶","劣","诬","耀","昏","躺","盈","骑","乔","溪","丛","卢","抹","易","闷","咨","刮","驾","缆","悟","摘","铒","掷","颇","幻","柄","惠","惨","佳","仇","腊","窝","涤","剑","瞧","堡","泼","葱","罩","霍","捞","胎","苍","滨","俩","捅","湘","砍","霞","邵","萄","疯","淮","遂","熊","粪","烘","宿","档","戈","驳","嫂","裕","徙","箭","捐","肠","撑","晒","辨","殿","莲","摊","搅","酱","屏","疫","哀","蔡","堵","沫","皱","畅","叠","阁","莱","敲","辖","钩","痕","坝","巷","饿","祸","丘","玄","溜","曰","逻","彭","尝","卿","妨","艇","吞","韦","怨","矮","歇","郊","禄","捻","漠","粹","颠","宏","冤","肪","饥","呵","仙","押","挨","醛","娃","拾","没","佩","勿","吓","讹","侯","恋","夕","锌","篡","戚","淋","蓬","岂","釉","兆","泊","魂","拘","亡","杠","摧","氟","颂","浑","凌","铀","诱","犁","谴","颁","舶","扯","嘉","萌","犹","滋","焊","舌","匹","媳","肺","掠","酿","烹","疲","驰","鸦","窄","辱","狭","朴","遣","菲","奸","韧","辣","拳","秆","卧","醉","竭","茅","墓","矣","哎","艳","敦","舆","缔","雇","尿","葬","履","契","禽","渣","衬","躲","赔","咸","溉","贼","醋","堤","抖","妃","裤","廉","晴","挽","掀","茫","丑","亥","拦","悠","阐","慧","佐","奇","竖","孝","柜","麟","绣","遥","逝","愁","肖","昭","芬","逢","窑","捷","圜","盲","闸","宙","辐","披","账","狼","幽","绸","蜂","慎","餐","酬","誓","惟","叉","弥","址","帜","芝","砌","唉","仆","涛","臭","翠","盒","劫","慨","炳","阖","寂","椒","倘","拓","畏","喉","巾","颈","垦","拚","兽","蔽","芦","乾","爽","窃","谭","挣","崩","模","褐","传","翅","儒","伞","晃","谬","胚","剖","凑","眠","浊","霜","礁","蔑","抄","闯","洒","碑","蓉","耶","猜","蹲","壶","唤","澳","锯","郡","玲","绵","纽","梳","掏","吁","锤","鼠","穴","椅","殷","遮","吵","萍","厌","畜","俱","夸","吕","囊","捧","雌","闽","饶","瞬","郁","哨","凿","朝","俺","浒","茂","肝","勋","盯","籽","耻","菊","滥","稼","戒","奈","帅","鞭","蚕","镁","询","跌","烤","坛","宅","笛","鄂","蛮","颤","棍","睁","鼎","岌","降","侍","藩","嚷","匪","岳","糟","缠","迪","泄","卑","氛","堪","萝","盛","碘","缚","悦","澄","甫","攀","屠","溢","拱","晰","携","朽","吟","菱","谦","凹","俊","芒","盼","婶","艘","酰","趁","唇","挫","羞","浴","疼","萎","肴","愚","肿","刨","绞","枢","嫁","慕","舱","铲","苹","豫","谕","迭","潘","顷","翁","榜","匠","欠","茬","畴","胃","沾","踪","弊","哼","鹏","歧","桐","沃","悼","惑","溃","蔗","荐","潭","孢","露","诊","庸","聪","嫌","厨","庞","祁","钳","肆","梭","赠","崖","篮","颖","甸","藻","捣","且","撕","诏","贞","赐","慈","炕","胖","兹","差","琼","锈","汛","卓","棵","馈","挠","灶","婴","蒂","肤","衫","沥","仑","勉","沪","逸","蜜","浦","嗓","晕","膏","祭","赢","艾","扮","鹅","怜","蒲","兔","孕","呖","蘖","挪","淑","谣","惧","廊","缅","俘","骄","膀","陡","宰","诞","峻","恼","腺","猎","涡","夷","愉","魔","铵","葛","贾","似","荫","哟","脊","钞","苛","锰","椭","镶","杏","溴","倚","滞","会","氓","捏","斩","傲","匆","僵","卤","烫","衍","榨","拢","裸","屑","咽","坊","舅","渴","翔","邪","拄","窖","猫","砌","钦","媒","脾","勺","柏","栅","噪","昼","耿","扁","辰","秤","得","贩","糕","梁","昙","衷","宦","扔","哇","诈","嘱","藤","卜","冈","悔","廓","皂","拐","氰","杉","玛","矢","寓","瓣","罕","垮","笋","淘","衔","称","恭","喇","帕","桉","秉","帘","铭","蛇","摔","斋","叭","帆","裸","俭","瘤","篷","砸","肢","辟","脖","瞪","暑","卜","竿","歼","笙","酮","蕴","哗","瞎","喀","刃","楔","喘","枚","嵌","挝","厢","粤","甩","拴","膝","恳","腕","娓","熄","锚","忌","愧","哦","荆","圃","骚","丸","蒜","毯","弗","俯","鹿","梢","屯","衙","轿","贱","垒","谅","踢","哑","滔","渥","饷","泳","棕","熬","搁","腈","梨","吻","樱","奠","捆","姨","柏","聘","惕","郓","绑","冀","裹","酥","寡","彦","稠","啡","钝","汝","擅","汰","鳙","埔","敞","嘿","逊","栋","谨","咖","鲤","雀","佣","庵","葫","贿","鳞","拼","搏","谎","塌","忉","腻","戊","怖","坟","禾","刹","嘻","桔","坎","拇","煽","狮","痒","曾","梗","寇","鹰","烛","哄","莽","雯","胳","龟","亟","糠","泌","坪","傻","什","喻","渊","蚌","跪","巷","涅","钊","譬","蕊","膛","侮","奕","枕","辫","况","扼","郝","寥","凄","厦","腥","钧","耦","蹄","戥","屁","诵","匈","桩","钓","涵","倦","袍","抒","屿","蹈","忿","敷","虹","聊","嗣","尉","灿","糙","蹬","嗯","姬","狡","笨","辜","僧","茨","讽","翰","枉","岐","枣","崭","焚","咕","猴","揽","涝","耍","趟","汹","咋","傍","镀","给","爵","虏","劈","璋","踩","瞅","迄","昔","汞","呱","诡","魄","祺","嘲","惶","赃","癌","咐","歉","扳","鄙","庐","聂","便","芡","躯","贬","煌","拧","隋","襄","淤","宠","炊","滇","謇","懒","栓","佑","憾","骆","裙","猖","兜","孵","痼","盥","曝","泣","絮","韵","眷","旷","噢","参","栖","盏","鳌","溅","煎","校","榴","暮","琪","淆","陛","巢","哒","吼","槐","唧","其","沛","乞","蜀","蜇","赚","捍","铰","幂","尧","咒","耽","叮","褂","焕","煞","雹","搓","釜","铬","拣","募","淹","瑰","鲢","茄","灼","邹","躬","觉","娇","焉","彰","鹤","琳","沦","畔","惹","庶","毙","皖","邢","禹","渍","绷","窜","翘","淫","箪","陌","膊","鞑","咳","玫","巫","拂","蕉","澜","赎","绥","锄","囱","赌","颊","缕","寅","躁","稚","庚","苟","氦","魁","珊","蜕","蛭","酌","逗","闺","蔓","撇","豌","朕","缉","襟","镍","桅","荧","侄","卒","佃","瞿","娶","饪","耸","乍","靶","痴","靖","扛","筐","韶","嚣","崔","蓿","岔","氘","娥","剿","霖","喃","搪","雍","裳","撰","豹","骏","慷"] static def totalCount = new AtomicLong() -// "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", -// "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", -// "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", -// "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", -// "呼和浩特":"hu|811", -// "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", -// "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", -// "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", -// "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", -// "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", -// "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", -// "淮北":"huaibei|9357","滁州":"chuzhou|10266", -// "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", -// "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", -// "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", -// "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", -// "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", -// "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", -// "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", -// "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", -// "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", -// "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", -// "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", -// "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", -// "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", -// "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", -// "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", -// "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", -// "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", -// "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", -// "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", -// "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", -// "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", -// "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", -// "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", -// "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", -// "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", -// "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", -// "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", -// "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", -// "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", -// "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", -// "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", -// "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", -// "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", -// "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", -// "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", -// "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", -// "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", -// "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", -// "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", -// "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", -// "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", -// "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", -// "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", -// "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", -// "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", -// "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", -// "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", -// "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", -// "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", -// "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", -// "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", -// "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", -// "邢台":"xt|751","衡水":"hs|993", -// "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", -// "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", -// "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", -// "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", -// "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", -// "沙河":"shaheshi|755", -// "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", -// "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", -// "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", -// "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", -// "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", -// "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", -// "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", -// "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", -// "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", -// "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", -// "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", -// "浮梁":"fuliangxian|9071", -// "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", -// "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", -// "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", -// "梨树县":"lishu|10176", -// "鞍山":"as|523","抚顺":"fushun|5722", -// "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", -// "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", -// "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", -// "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", -// "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", -// "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", -// "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", -// "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", -// "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", -// "泰安":"ta|686","聊城":"lc|882", -// "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", -// "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", -// "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", -// "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", -// "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", -// "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", -// "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", -// "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", -// "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", -// "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", -// "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", -// "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", -// "临朐":"linqu|374","鄄城":"juancheng|5635", -// "运城":"yuncheng|5653","晋中":"jz|8854", -// "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", -// "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", -// "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", -// "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", -// "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", -// "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", -// "德阳":"deyang|2373","宜宾":"yb|2380", -// "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", -// "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", -// "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", -// "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", -// "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", -// "昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", -// "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", -// "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", -// "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", -// "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", -// "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", -// "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", -// "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", -// "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", -// "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", -// "丽水":"lishui|7921","衢州":"quzhou|6793", -// "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", -// "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", -// "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", -// "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", -// "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", -// "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" + def cityList = [ "福州":"fz|304","蚌埠":"bengbu|3470","淮南":"hn|2319","青岛":"qd|122","广州":"gz|3","天津":"tj|18","南京":"nj|172", "嘉兴":"jx|497","大连":"dl|147","武汉":"wh|158", - "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", - "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", - "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", - "呼和浩特":"hu|811", - "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", - "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", - "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", - "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", - "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", - "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", - "淮北":"huaibei|9357","滁州":"chuzhou|10266", - "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", - "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", - "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", - "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", - "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", - "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", - "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", - "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", - "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", - "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", - "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", - "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", - "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", - "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", - "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", - "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", - "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", - "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", - "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", - "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", - "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", - "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", - "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", - "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", - "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", - "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", - "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", - "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", - "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", - "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", - "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", - "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", - "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", - "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", - "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", - "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", - "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", - "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", - "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", - "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", - "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", - "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", - "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", - "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", - "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", - "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", - "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", - "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", - "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", - "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", - "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", - "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", - "邢台":"xt|751","衡水":"hs|993", - "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", - "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", - "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", - "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", - "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", - "沙河":"shaheshi|755", - "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", - "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", - "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", - "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", - "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", - "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", - "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", - "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", - "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", - "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", - "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", - "浮梁":"fuliangxian|9071", - "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", - "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", - "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", - "梨树县":"lishu|10176", - "鞍山":"as|523","抚顺":"fushun|5722", - "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", - "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", - "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", - "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", - "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", - "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", - "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", - "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", - "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", - "泰安":"ta|686","聊城":"lc|882", - "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", - "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", - "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", - "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", - "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", - "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", - "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", - "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", - "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", - "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", - "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", - "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", - "临朐":"linqu|374","鄄城":"juancheng|5635", - "运城":"yuncheng|5653","晋中":"jz|8854", - "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", - "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", - "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", - "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", - "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", - "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", - "德阳":"deyang|2373","宜宾":"yb|2380", - "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", - "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", - "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", - "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", - "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", - "昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", - "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", - "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", - "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", - "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", - "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", - "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", - "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", - "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", - "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", - "丽水":"lishui|7921","衢州":"quzhou|6793", - "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", - "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", - "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", - "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", - "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", - "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" + "兰州":"lz|952","合肥":"hf|837", "乌鲁木齐":"xj|984","南昌":"nc|669","沈阳":"sy|188","西安":"xa|483","苏州":"su|5","无锡":"wx|93","潍坊":"wf|362", + "芜湖":"wuhu|2045","东莞":"dg|413","常州":"cz|463","佛山":"fs|222", + "南通":"nt|394","东莞":"dg|413","吉林":"jl|700","中山":"zs|771","阜阳":"fy|2325", + "呼和浩特":"hu|811", + "济南":"jn|265","温州":"wz|330","台州":"tz|403","东营":"dy|623","咸阳":"xianyang|7453","安庆":"anqing|3251","宿州":"suzhou|3359","六安":"la|2328","唐山":"ts|276","商丘":"sq|1029","大同":"dt|6964","威海":"weihai|518","安阳":"ay|1096", + "临汾":"linfen|5669","临沂":"linyi|505","九江":"jj|2247","淄博":"zb|385","北京":"bj|1","上海":"sh|2", + "三亚":"sanya|2422","烟台":"yt|228","宁波":"nb|135","金华":"jh|531", "绍兴":"sx|355","湖州":"huzhou|831","张家口":"zjk|3328","珠海":"zh|910","海口":"haikou|2053","保定":"bd|424","保亭":"baoting|10367","南宁":"nn|845", "扬州":"yz|637","汕头":"st|783","泰州":"taizhou|693","淮安":"ha|968","湛江":"zhanjiang|791","锦州":"jinzhou|2354","昆明":"km|541","柳州":"liuzhou|7133","肇庆":"zq|901","连云港":"lyg|2049","日照":"rizhao|3177","常德":"changde|872", + "沧州":"cangzhou|652", "泉州":"qz|291","清远":"qingyuan|7303", "西宁":"xn|2052","遵义":"zunyi|7620","株洲":"zhuzhou|1086","贵阳":"gy|2015","岳阳":"yy|821","徐州":"xz|471", "哈尔滨":"hrb|202","宜昌":"yc|858", "廊坊":"lf|772","秦皇岛":"qhd|1078","惠州":"huizhou|722","洛阳":"luoyang|556", + "深圳":"sz|4","石家庄":"sjz|241","太原":"ty|740","成都":"cd|102","杭州":"hz|79","重庆":"cq|37","盐城":"yancheng|613","信阳":"xy|8694","南充":"nanchong|2378","南阳":"ny|592","厦门":"xm|606", + "绵阳":"mianyang|1057","济宁":"jining|450","邯郸":"hd|572", + "淮北":"huaibei|9357","滁州":"chuzhou|10266", + "马鞍山":"mas|2039","铜陵":"tongling|10285","宣城":"xuancheng|5633","亳州":"bozhou|2329","黄山":"huangshan|2323", + "池州":"chizhou|10260","巢湖":"ch|10229","和县":"hexian|10892","霍邱":"hq|11226","桐城":"tongcheng|11296", + "宁国":"ningguo|5645","天长":"tianchang|10273","东至":"dongzhi|10262","无为":"wuweixian|10232", + "莆田":"pt|2429","漳州":"zhangzhou|710","宁德":"nd|7951","三明":"sm|2048","南平":"np|10291","龙岩":"ly|6752","武夷山":"wuyishan|10761", + "石狮":"shishi|296","晋江":"jinjiangshi|297","南安":"nananshi|293","龙海":"longhai|713","上杭":"shanghangxian|6757", + "福安":"fuanshi|7969","福鼎":"fudingshi|7970","安溪":"anxixian|7100","永春":"yongchunxian|7101","永安":"yongan|2133", + "漳浦":"zhangpu|717", "珠海":"zh|910","江门":"jm|629", + "茂名":"mm|679","揭阳":"jy|927","梅州":"mz|9389", + "阳江":"yj|2284","韶关":"sg|2192","河源":"heyuan|10467","云浮":"yf|10485","汕尾":"sw|9449", + "潮州":"chaozhou|10461","台山":"taishan|11263","阳春":"yangchun|8566","顺德":"sd|8716","惠东":"huidong|725", + "博罗":"boluo|726","海丰":"haifengxian|9444","开平":"kaipingshi|634","陆丰":"lufengshi|9456", + "桂林":"gl|1039","玉林":"yulin|2337","梧州":"wuzhou|2046", + "北海":"bh|10536","贵港":"gg|6770","钦州":"qinzhou|2335","百色":"baise|10513","河池":"hc|2340", + "来宾":"lb|10552","贺州":"hezhou|10549","防城港":"fcg|10539","崇左":"chongzuo|10524","桂平":"guipingqu|6774", + "北流":"beiliushi|9168","博白":"bobaixian|9173","岑溪":"cenxi|2119", + "黔东南":"qdn|9363","黔南":"qn|10492","六盘水":"lps|10506", + "毕节":"bijie|10564","铜仁":"tr|10417","安顺":"anshun|7468","黔西南":"qxn|10434","仁怀":"renhuaishi|7628", + "清镇":"qingzhen|12703","天水":"tianshui|8601","白银":"by|10304","庆阳":"qingyang|10475","平凉":"pl|7154", + "酒泉":"jq|10387","张掖":"zhangye|10454","武威":"wuwei|10448","定西":"dx|10322","金昌":"jinchang|7428", + "陇南":"ln|10415","临夏":"linxia|7112","嘉峪关":"jyg|10362","甘南":"gn|10343","敦煌":"dunhuang|10390", + "五指山":"wzs|9952","三沙":"sansha|13722","琼海":"qh|10136", + "文昌":"wenchang|9984","万宁":"wanning|10022","屯昌":"tunchang|10044","琼中":"qiongzhong|10064","陵水":"lingshui|10184", + "东方":"df|10250","定安":"da|10303","澄迈":"cm|10331","白沙":"baish|10380", + "儋州":"danzhou|10394", "郑州":"zz|342","新乡":"xx|1016","许昌":"xc|977", + "平顶山":"pds|1005","焦作":"jiaozuo|3266","开封":"kaifeng|2342", + "濮阳":"puyang|2346","周口":"zk|933","驻马店":"zmd|1067","漯河":"luohe|2347", + "三门峡":"smx|9317","鹤壁":"hb|2344","济源":"jiyuan|9918","明港":"mg|8541","鄢陵":"yanling|9123", + "禹州":"yuzhou|979","长葛":"changge|9344","灵宝":"lingbaoshi|9307","杞县":"qixianqu|7389","汝州":"ruzhou|1010", + "项城":"xiangchengshi|935","偃师":"yanshiqu|7121","长垣":"changyuan|5936","滑县":"huaxian|5405","林州":"linzhou|1101", + "沁阳":"qinyang|3268","孟州":"mengzhou|3267","温县":"wenxian|7312","尉氏":"weishixian|7391","兰考":"lankaoxian|7393", + "通许":"tongxuxian|7390","新安":"lyxinan|11217","伊川":"yichuan|11220","孟津":"mengjinqu|7122","宜阳":"lyyiyang|11219", + "舞钢":"wugang|1011","永城":"yongcheng|1032","睢县":"suixian|1038","鹿邑":"luyi|939","渑池":"yingchixian|9322", + "沈丘":"shenqiu|942","太康":"taikang|938","商水":"shangshui|936","淇县":"qixianq|9186","浚县":"junxian|9185", + "范县":"fanxian|7285","固始":"gushixian|8698","淮滨":"huaibinxian|8702","邓州":"dengzhou|595","新野":"xinye|603", + "大庆":"dq|375","齐齐哈尔":"qqhr|5853","牡丹江":"mdj|3489","绥化":"suihua|6718", + "佳木斯":"jms|6776","鸡西":"jixi|7289","双鸭山":"sys|9837","鹤岗":"hegang|9061","黑河":"heihe|9862", + "伊春":"yich|9773","七台河":"qth|9848","大兴安岭":"dxal|9878","安达":"shanda|6720","肇东":"shzhaodong|6721", + "肇州":"zhaozhou|382", "襄阳":"xf|891","荆州":"jingzhou|3479","十堰":"shiyan|2032", + "黄石":"hshi|1734","孝感":"xiaogan|3434","黄冈":"hg|2299","恩施":"es|2302","荆门":"jingmen|2296", + "咸宁":"xianning|9617","鄂州":"ez|9709","随州":"suizhou|9656","潜江":"qianjiang|9669","天门":"tm|9517", + "仙桃":"xiantao|9736","神农架":"snj|9605","宜都":"yidou|864","汉川":"hanchuan|3439","枣阳":"zaoyang|896", + "武穴":"wuxueshi|7362","钟祥":"zhongxiangshi|9119","京山":"jingshanxian|9117","沙洋":"shayangxian|9118","松滋":"songzi|3484", + "广水":"guangshuishi|9657","赤壁":"chibishi|9623","老河口":"laohekou|895","谷城":"gucheng|899","宜城":"yichengshi|897", + "南漳":"nanzhang|898","云梦":"yunmeng|3438","安陆":"anlu|3442","大悟":"dawu|3437","孝昌":"xiaochang|3436", + "当阳":"dangyang|865","枝江":"zhijiang|866","嘉鱼":"jiayuxian|9624","随县":"suixia|9660", + "长沙":"cs|414","益阳":"yiyang|10198","衡阳":"hy|914", + "湘潭":"xiangtan|2047","郴州":"chenzhou|5695","邵阳":"shaoyang|2303","怀化":"hh|5756", + "永州":"yongzhou|2307","娄底":"ld|9481","湘西":"xiangxi|10219","张家界":"zjj|6788","醴陵":"liling|1091", + "澧县":"lixian|876","桂阳":"czguiyang|5699","资兴":"zixing|5698","永兴":"yongxing|5701","常宁":"changningshi|921", + "祁东":"qidongxian|5690","衡东":"hengdong|5693","冷水江":"lengshuijiangshi|9470","涟源":"lianyuanshi|9471","双峰":"shuangfengxian|9473", + "邵阳县":"shaoyangxian|6955","邵东":"shaodongxian|6954","沅江":"yuanjiangs|10201","南县":"nanxian|10202","祁阳":"qiyang|8532", + "湘阴":"xiangyin|828","华容":"huarong|830","慈利":"cilixian|6791","攸县":"zzyouxian|1095", + "邢台":"xt|751","衡水":"hs|993", + "承德":"chengde|6760","定州":"dingzhou|8398","馆陶":"gt|8706","张北":"zhangbei|11201","赵县":"zx|9048", + "正定":"zd|3198","迁安市":"qianan|284","任丘":"renqiu|656","三河":"sanhe|776","武安":"wuan|577", + "雄安新区":"xionganxinqu|111234","燕郊":"lfyanjiao|12804","涿州":"zhuozhou|428","河间":"hejian|658","黄骅":"huanghua|657", + "沧县":"cangxian|659","磁县":"cixian|591","涉县":"shexian|14059","霸州":"bazhou|775","香河":"xianghe|5395", + "固安":"lfguan|12803","遵化市":"zunhua|283","迁西":"qianxixian|7061","玉田":"yutianxian|7060","滦南":"luannanxian|7066", + "沙河":"shaheshi|755", + "宿迁":"suqian|2350","镇江":"zj|645","沭阳":"shuyang|5772","大丰":"dafeng|11279", + "如皋":"rugao|397","启东":"qidong|400","溧阳":"liyang|469","海门":"haimen|399","东海":"donghai|2147", + "扬中":"yangzhong|649","兴化":"xinghuashi|699","新沂":"xinyishi|478","泰兴":"taixing|696","如东":"rudong|402", + "邳州":"pizhou|477","沛县":"xzpeixian|11349","靖江":"jingjiang|698","建湖":"jianhu|618","海安":"haian|401", + "东台":"dongtai|615","丹阳":"danyang|648","宝应县":"baoyingx|14451","灌南":"guannan|2150","灌云":"guanyun|2148", + "姜堰":"jiangyan|697","金坛":"jintan|468","昆山":"szkunshan|16","泗洪":"sihong|5958","泗阳":"siyang|5959", + "句容":"jurong|650","射阳":"sheyang|621","阜宁":"funingxian|620","响水":"xiangshui|619","盱眙":"xuyi|976", + "金湖":"jinhu|975","江阴":"jiangyins|34984", "赣州":"ganzhou|2363","宜春":"yichun|5709","吉安":"ja|2364", + "上饶":"sr|10120","萍乡":"px|2248","抚州":"fuzhou|10134","景德镇":"jdz|2360","新余":"xinyu|10115", + "鹰潭":"yingtan|3209","永新":"yxx|11077","乐平":"lepingshi|9048","进贤":"jinxian|677","分宜":"fenyi|10118", + "丰城":"fengchengshi|5711","樟树":"zhangshu|5713","高安":"gaoan|5712","余江":"yujiang|3210","南城":"nanchengx|10137", + "浮梁":"fuliangxian|9071", + "长春":"cc|319","四平":"sp|10171","延边":"yanbian|3184","松原":"songyuan|2315", + "白城":"bc|5918","通化":"th|10159","白山":"baishan|10179","辽源":"liaoyuan|2501","公主岭":"gongzhuling|10171", + "梅河口":"meihekou|10162","扶余":"fuyuxian|9085","长岭":"changlingxian|9084","桦甸":"huadian|706","磐石":"panshi|708", + "梨树县":"lishu|10176", + "鞍山":"as|523","抚顺":"fushun|5722", + "营口":"yk|5898","盘锦":"pj|2041","朝阳":"cy|10106","丹东":"dandong|3445","辽阳":"liaoyang|2038", + "本溪":"benxi|5845","葫芦岛":"hld|10088","铁岭":"tl|6729","阜新":"fx|10097","庄河":"pld|3306", + "瓦房店":"wfd|3279","灯塔":"dengta|2071","凤城":"fengcheng|3450","北票":"beipiao|10109","开原":"kaiyuan|6733", + "银川":"yinchuan|2054","吴忠":"wuzhong|9962","石嘴山":"szs|9971","中卫":"zw|9951","固原":"guyuan|2421", + "包头":"bt|801","赤峰":"chifeng|6700","鄂尔多斯":"erds|2037","通辽":"tongliao|10015", + "呼伦贝尔":"hlbe|10039","巴彦淖尔市":"bycem|10070","乌兰察布":"wlcb|9993","锡林郭勒":"xl|2408","兴安盟":"xam|9976", + "乌海":"wuhai|2404","阿拉善盟":"alsm|10083","海拉尔":"hlr|2043", + "海西":"hx|9902","海北":"haibei|9917","果洛":"guoluo|9936","海东":"haidong|9909", + "黄南":"huangnan|9896","玉树":"ys|9888","海南":"hainan|10574","格尔木":"geermushi|9904", + "泰安":"ta|686","聊城":"lc|882", + "枣庄":"zaozhuang|961","德州":"dz|728","菏泽":"heze|5632", + "滨州":"bz|944","莱芜":"lw|2292","章丘":"zhangqiu|8680","垦利":"kl|11313","诸城":"zc|9146", + "寿光":"shouguang|369","龙口":"longkou|233","曹县":"caoxian|5638","单县":"shanxian|5636","肥城":"feicheng|690", + "高密":"gaomi|371","广饶":"guangrao|627","桓台":"huantaixian|7335","莒县":"juxian|3180","莱州":"laizhou|235", + "蓬莱":"penglai|237","青州":"qingzhou|367","荣成":"rongcheng|522","乳山":"rushan|520","滕州":"tengzhou|967", + "新泰":"xintai|689","招远":"zhaoyuan|3325","邹城":"zoucheng|455","邹平":"zouping|946","临清":"linqing|884", + "茌平":"chiping|887","郓城":"hzyc|5637","博兴":"boxing|949","东明":"dongming|5641","巨野":"juye|5640", + "无棣":"wudi|951","齐河":"qihe|734","微山":"weishan|459","禹城":"yuchengshi|731","临邑":"linyixianq|739", + "乐陵":"leling|730","莱阳":"laiyang|234","宁津":"ningjin|733","高唐":"gaotang|885","莘县":"shenxian|888", + "阳谷":"yanggu|886","冠县":"guanxian|890","平邑":"pingyi|514","郯城":"tancheng|510","沂源":"yiyuanxian|7334", + "汶上":"wenshang|460","梁山":"liangshanx|462","利津":"lijin|628","沂南":"yinanxian|7301","栖霞":"qixia|238", + "宁阳":"ningyang|691","东平":"dongping|692","昌邑":"changyishi|372","安丘":"anqiu|370","昌乐":"changle|373", + "临朐":"linqu|374","鄄城":"juancheng|5635", + "运城":"yuncheng|5653","晋中":"jz|8854", + "长治":"changzhi|6921","晋城":"jincheng|3350","阳泉":"yq|8760","吕梁":"lvliang|3222","忻州":"xinzhou|3453", + "朔州":"shuozhou|9871","临猗":"linyixian|9193","清徐":"qingxu|10908","柳林":"liulin|3225","高平":"gaoping|3354", + "泽州":"zezhou|3353","襄垣":"xiangyuanxian|6928","孝义":"xiaoyi|3227", + "宝鸡":"baoji|2044","渭南":"wn|5733","汉中":"hanzhong|3163", + "榆林":"yl|5942","延安":"yanan|8973","安康":"ankang|3157","商洛":"sl|9854","铜川":"tc|9832", + "神木":"shenmu|5944","韩城":"hancheng|5735","府谷":"fugu|5945","靖边":"jingbian|5947","定边":"dingbian|5948", + "德阳":"deyang|2373","宜宾":"yb|2380", + "自贡":"zg|6745","乐山":"ls|3237","泸州":"luzhou|2372","达州":"dazhou|9799","内江":"scnj|5928", + "遂宁":"suining|9688","攀枝花":"panzhihua|2371","眉山":"ms|9704","广安":"ga|2381","资阳":"zy|6803", + "凉山":"liangshan|9717","广元":"guangyuan|9749","雅安":"ya|9687","巴中":"bazhong|9811","阿坝":"ab|9817", + "甘孜":"ganzi|9764","安岳":"anyuexian|6806","广汉":"guanghanshi|8719","简阳":"jianyangshi|6805","仁寿":"renshouxian|9706", + "射洪":"shehongxian|9694","大竹":"dazu|9806","宣汉":"xuanhan|9804","渠县":"qux|9807","长宁":"changningx|7148", + "昌吉":"changji|8582","巴音郭楞":"bygl|9530","伊犁":"yili|9472","阿克苏":"aks|9499", + "喀什":"ks|9326","哈密":"hami|7452","克拉玛依":"klmy|2042","博尔塔拉":"betl|9529","吐鲁番":"tlf|9475", + "和田":"ht|9489","石河子":"shz|9551","克孜勒苏":"kzls|9519","阿拉尔":"ale|9539","五家渠":"wjq|9562", + "图木舒克":"tmsk|9559","库尔勒":"kel|7168","阿勒泰":"alt|18837","塔城":"tac|18845", + "拉萨":"lasa|2055","日喀则":"rkz|9615","山南":"sn|9576","林芝":"linzhi|9646","昌都":"changdu|9648", + "那曲":"nq|9618","阿里":"al|9678","日土":"rituxian|9682","改则":"gaizexian|9684", + "曲靖":"qj|2389","大理":"dali|2398","红河":"honghe|2394","玉溪":"yx|2040", + "丽江":"lj|2392","文山":"ws|2395","楚雄":"cx|2393","西双版纳":"bn|2397","昭通":"zt|9409", + "德宏":"dh|9437","普洱":"pe|9444","保山":"bs|2390","临沧":"lincang|9422","迪庆":"diqing|9432", + "怒江":"nujiang|9462","弥勒":"milexian|8892","安宁":"anningshi|547","宣威":"xuanwushi|7533", + "丽水":"lishui|7921","衢州":"quzhou|6793", + "舟山":"zhoushan|8481","乐清":"yueqingcity|13950","瑞安":"ruiancity|13951","义乌":"yiwu|12291","余姚":"yuyao|5333", + "诸暨":"zhuji|357","象山":"xiangshanxian|6738","温岭":"wenling|408","桐乡":"tongxiang|502","慈溪":"cixi|5334", + "长兴":"changxing|834","嘉善":"jiashanx|14357","海宁":"haining|500","德清":"deqing|835","东阳":"dongyang|536", + "安吉":"anji|836","苍南":"cangnanxian|7579","临海":"linhai|407","永康":"yongkang|537","玉环":"yuhuan|409", + "平湖":"pinghushi|501","海盐":"haiyan|504","武义县":"wuyix|14528","嵊州":"shengzhou|359","新昌":"xinchang|361", + "江山":"jiangshanshi|6796","平阳":"pingyangxian|7575" ] static main(args) { @@ -331,11 +191,24 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["郑州"] - cityList.each{city, str-> + def citys = ["成都","重庆","杭州","深圳","郑州","长沙","哈尔滨","长春","石家庄","太原","惠州","厦门","洛阳","德阳","镇江","泰安","宜昌","银川","襄阳","江门","荆州","滨州","衡阳","遂宁","鞍山","赣州","聊城","枣庄","德州","宿迁","新乡","信阳","包头","乐山","桂林","眉山","驻马店","许昌","泸州","宜宾","衡水","营口","达州","广安","吉安","鄂尔多斯","孝感","宝鸡","湘潭","十堰","上饶","菏泽","漯河","邢台","承德","宜春","平顶山","赤峰","延边","韶关","内江","茂名","梅州","焦作","郴州","黄冈","红河","绥化","丹东","阳江","娄底","黄石","开封","邵阳","牡丹江","周口","自贡","咸宁","汉中","长治","大庆","抚顺","北海","衢州","呼伦贝尔","永州","濮阳","晋中","晋城","河源","渭南","巴中","运城","舟山","齐齐哈尔","曲靖","怀化","铁岭","宁德","佳木斯","朝阳","葫芦岛","资阳","鄂州","通辽","益阳","荆门","大理","辽阳","阳泉","榆林","梧州","黔南","乌兰察布","昌吉","巴音郭楞","玉溪"] + for(def city : citys){ + def str = cityList.get(city) for(def i=0;i<1;i++){ -// def city = "福州" -// if(citys.contains(city)){ + def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); + if(districts){ + def arrs = str.split("\\|") + def url = "https://"+arrs[0]+".58.com/xiaoqu/" + distMap.put(city, districts) + addCrawlKeys(city); + addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); + } + } + } + + /*cityList.each{city, str-> + for(def i=0;i<1;i++){ + if(citys.contains(city)){ def districts = CommonDatautils.findAllDistricts(city, "district", ["city", "region", "name"]); if(districts){ def arrs = str.split("\\|") @@ -344,12 +217,10 @@ class Esf58Crawl extends BaseBatch{ addCrawlKeys(city); addDetailContent(city, "https://"+arrs[0]+".58.com", arrs[1]); - -// Thread.sleep(1000*60*10) } -// } + } } - } + }*/ } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 2d055dcbb..1b6e24286 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -18,7 +18,7 @@ class SynchDistrictPrice { // def sourceMap = ["fang.com":"搜房","jiwu.com":"吉屋","58.com":"58同城","lianjia.com":"链家","ke.com":"贝壳找房","fangdd.com":"房多多","house365.com":"365淘房", // "anjuke.com":"安居客","cityhouse.cn":"城市房产","zhuge.com":"诸葛找房","5i5j.com":"我爱我家"] //"fang.com":"搜房", - def sourceMap = ["cityhouse.cn":"城市房产"] + def sourceMap = ["jiwu.com":"吉屋","ke.com":"贝壳找房","cityhouse.cn":"城市房产"] static main(args) { def task = new SynchDistrictPrice() task.run() From 860ffad413fb237a4e1fc131a39ca365ce0ab553 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Fri, 11 Mar 2022 15:33:16 +0800 Subject: [PATCH 203/212] update --- .../ccbevaluation/data/ExcelDataTest.groovy | 53 ++++-- .../com/fangjia/script/fuzhou/Test1.groovy | 2 +- .../chongqingblank/DistrictSelectHouse.groovy | 163 ++++++++++++++++++ 3 files changed, 203 insertions(+), 15 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DistrictSelectHouse.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index 1df8657c9..8fc309322 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -11,6 +11,7 @@ import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection +import com.mongodb.DBObject import groovy.sql.Sql @@ -454,22 +455,46 @@ def city = " // } //}) -ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d -> +//ExcelUtils.modify("D:\\数据\\小区\\评级小区.xls", { d -> +// if(d['A']){ +// try{ +// def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"]) +// if(dist){ +// d['B'] = dist.city +// d['C'] = dist.region +// d['D'] = dist.name +// +// def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name) +// d['E'] = price +// } +// }catch(Exception e){ +// println d['A'] +// } +// +// } +// +//}) + +ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d -> if(d['A']){ - try{ - def dist = CommonDatautils.findDistrictFieldById(d['A'],["city","region","name"]) - if(dist){ - d['B'] = dist.city - d['C'] = dist.region - d['D'] = dist.name - - def price = CommonDatautils.queryDistrictPrice(dist.city, dist.region, dist.name) - d['E'] = price + def dist = CommonDatautils.findDistrictById(d['A']) + if(dist){ + def mappingAlias = dist.mapping_alias + if(mappingAlias){ + if(mappingAlias.contains(d['B'])){ + mappingAlias.remove(d['B']) + DBObject udata = new BasicDBObject(); + udata.put("mapping_alias", mappingAlias); + udata.put("m_date", new Date()); + DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata)) + }else{ + println d['A']+"无映射别名" + } + }else{ + println d['A']+"映射别名为空" } - }catch(Exception e){ - println d['A'] + }else{ + println d['A']+"没找到" } - } - }) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy index 824cbda57..a8a616b7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/Test1.groovy @@ -14,7 +14,7 @@ import com.mongodb.DBCollection //福州第三方均价统计,发送钉钉消息 //def datas = ExcelUtils.read("d:\\数据\\福州\\福州4月2.xlsx",0,1) //def datas = CsvUtils.read("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202105.csv"); -def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202201.csv") +def datas = readTxt("d:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv") def count0 = 0 def count1 = 0 def count2 = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DistrictSelectHouse.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DistrictSelectHouse.groovy new file mode 100644 index 000000000..ddcfd51fc --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/chongqingblank/DistrictSelectHouse.groovy @@ -0,0 +1,163 @@ +package com.fangjia.script.gj.chongqingblank + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DataUtil +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.QueryOperators +import com.fangjia.core.util.DataUtil +import org.apache.commons.lang.math.RandomUtils + +class DistrictSelectHouse { + + DBCollection DIST_SELL_STAT = MongoUtils.getCollectionByName("fangjia_stat", "district_stat", "mongo.db.fangjia.com") + static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + + static main(args) { + def task = new DistrictSelectHouse(); +// task.run() +// task.run2() + task.run3() + } + + def run3(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本V1 -跑地址.xlsx",{d-> + if(d['A']) { + def dist = CommonDatautils.findDistrict(d['A'],d['B'], d['C'], "district") + if(dist){ + d['L'] = dist.address + } + } + },0) + } + + def run2(){ + ExcelUtils.multiThreadingModify("D:\\数据\\重庆农商行\\重庆样本.xlsx",{d-> + if(d['N']){ + def floorInfo = d['J'] + def height = d['N'] + if(floorInfo && height){ + height = height.toInteger() + def mod = (int)(height/3) + def h_start = (height - mod).toInteger() + def random_floor = null + if(floorInfo.contains("低楼层")){ + random_floor = random(1, mod) + }else if(floorInfo.contains("中楼层")){ + random_floor = random(mod+1, h_start-1) + }else if(floorInfo.contains("高楼层")){ + random_floor = random(h_start, height) + } + d['M'] = random_floor + } + } + },0) + + } + + + int random(int i1, int i2) { + int max = Math.max(i1, i2); + int min = Math.min(i1, i2); + return min + RandomUtils.nextInt(max - min + 1); + } + + + def run(){ + def city = "重庆" + def weekend = "2022-03-06" + def thirdPriceMap = findNearThirdPartyPrice(city) + def distStats = findCreditHightDistrict2(city, weekend, thirdPriceMap) + findAllHouse(distStats); + } + + def findAllHouse(distStats){ + def houses = [] + distStats.each{k,v-> + def arrs = k.split("#") + def list = findHouses(arrs[0],arrs[1], arrs[2], 20220201,20220401) + if(list){ + houses.addAll(list) + } + } + if(houses){ + def rets = [] + for(def h : houses){ + def row = [] + row << h.city + row << h.region + row << h.district_name + row << h.area + row << h.avg_price + row << h.total_price + row << h.direction + row << h.estate_type + row << h.fitment + row << h.floor_info + row << h.room + row << h.hall + row << h.floor + row << h.height + row << h.p_date + rets << row + } + ExcelUtils.write("D:\\数据\\重庆农商行\\重庆样本.xlsx", rets, "数据") + } + } + + + def findHouses(def city, def region, def name, def startDate, def endDate){ + String sql = "select * from latest_seawater where cat='sell' and root='lianjia.com' and city='${city}' and region='${region}' and district_name='${name}' and p_date >= ${startDate} and p_date<${endDate}" + def houses = [] + ElasticUtils.executeSqlQuery(sql,{ data-> + houses << data + }) + if(houses.size()>10){ + houses = houses[0..9] + } + return houses + } + + + def findNearThirdPartyPrice(def city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', 202108)) + q.put("lianjia_esf_price", new BasicDBObject(QueryOperators.EXISTS, true)) + def dbcursor = AVERAGE_STATISTICS.find(q) + def thirdPriceMap = [:] + for(def d : dbcursor){ + def key = d.city+"#"+d.region+"#"+d.name + if(!thirdPriceMap.containsKey(key)){ + thirdPriceMap.put(d.city+"#"+d.region+"#"+d.name, d.lianjia_esf_price) + } + } + return thirdPriceMap + } + + + + def findCreditHightDistrict2(def city, def weekend, def thirdPriceMap){ + def q = new BasicDBObject() + q.put("city", city) + q.put("value_type", 2) + q.put("weekend", DateUtils.parseFromTightByPattern(weekend, "yyyy-MM-dd")) + def dbcursor = DIST_SELL_STAT.find(q).sort(new BasicDBObject("credit",-1)) + def retMaps = [:] + for(def d : dbcursor){ + def thirdPrice = thirdPriceMap.get(d.city +"#"+d.region+"#"+d.district_name) + if(thirdPrice) { + def precent = (thirdPrice - d.avg_price) * 100 / d.avg_price + if (Math.abs(precent) >= 0 && Math.abs(precent) <= 10) { + retMaps.put(d.city +"#"+d.region+"#"+d.district_name, d.avg_price) + } + } + } + return retMaps + } + +} From 677292c7361e6b70b09b21eb3ab03ef32da5e3e5 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Sun, 13 Mar 2022 12:05:01 +0800 Subject: [PATCH 204/212] =?UTF-8?q?year=E6=94=B9=E4=B8=BA2021?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy index c7a92ae8e..c1a7d36eb 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/temp/shenjian/poi/CommitteeLatLng.groovy @@ -21,10 +21,10 @@ import java.util.concurrent.Executors class CommitteeLatLng { static DBCollection villagesConn = MongoUtils.getCollectionByName("fangjia", "villages", "192.168.0.43", 28017) - static ExecutorService executorService = Executors.newFixedThreadPool(20) + static ExecutorService executorService = Executors.newFixedThreadPool(10) static void main(String[] args) { - dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("name", "席庄二村委会")) + dealPoi(new BasicDBObject("status_location", new BasicDBObject('$ne', 0)).append("year", 2021)) //dealLocation() executorService.shutdown() } From 24cddd20fe3fb79a0061ac088b6064921c3897f0 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Mar 2022 13:59:14 +0800 Subject: [PATCH 205/212] =?UTF-8?q?=E6=88=BF=E6=BA=90=E6=97=B6=E9=97=B4?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/BigdataForsaleBatch.groovy | 2 +- .../script/ccbevaluation/TradeStatLog.groovy | 16 +- .../ccbevaluation/data/ExcelDataTest.groovy | 144 +++++++-- .../ccbevaluation/data/MatchMapData.groovy | 50 +++- .../data/TradeDistrictToDB.groovy | 198 +++++++++++++ .../script/commData/CommonDatautils.groovy | 18 +- .../fangjia/script/fuzhou/ThirdTest.groovy | 278 ++++++++++++++++++ .../fangjia/script/gj/YouDaDataParser.groovy | 11 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 8 +- .../script/gj/paimai/GaoDeUtils.groovy | 3 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 7 +- .../script/thirdPriceCrawl/DataStat.groovy | 6 +- .../script/thirdPriceCrawl/Esf58Crawl.groovy | 30 +- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 4 +- 14 files changed, 707 insertions(+), 68 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TradeDistrictToDB.groovy create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdTest.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy index b715ccdde..2749ed7f4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/BigdataForsaleBatch.groovy @@ -74,7 +74,7 @@ class BigdataForsaleBatch extends BaseBatch{ // def distIds = getDistrictIds(cityCodeDataMap.get(city)) def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) def districts = CcbCommonUtils.findAllDistrictByCity(city, "district", ["city", "region", "name", "lng","estate_type2","name_cat","address", "alias", "finish_building_date", "house_property", "property_life", "build_type", "build_structure"]); - exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220118, 20220219); + exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20220218, 20220319); // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20210801, 20210831); //测试通过匹配接口配房源效果 // exportSellCSV(index, city, cityCodeDataMap, distCodeMap, districts, 20201201, 20211118) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 621eac1ee..778b9c5ea 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,7 +40,7 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() -// task.run()//统计最近3个月抓取的成交数据量 + task.run()//统计最近3个月抓取的成交数据量 // task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) @@ -49,7 +49,7 @@ class TradeStatLog { // task.deleteDealData() // task.deleteDealData2() // task.statTableCount() - task.filterTrade() +// task.filterTrade() } @@ -75,13 +75,13 @@ class TradeStatLog { def run(){ def citys = CcbCommonUtils.citys; def rets = [] - rets << ["城市","11月总量","12月总量","1月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] + rets << ["城市","12月总量","1月总量","2月总量","占比","贝壳","链家", "我爱我家","房天下","中原"] def sources = ["贝壳找房","链家在线", "我爱我家","房天下","中原地产"] //def startDate = "2021-05-01" - def startDate = "2021-11-01" - def date = "2021-12-01" - def endDate = "2022-01-01" - def newDate = "2022-02-01" + def startDate = "2021-12-01" + def date = "2022-01-01" + def endDate = "2022-02-01" + def newDate = "2022-03-01" for(def city : citys){ def lastMonthCount = findTradesCount(city, null, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(date,"yyyy-MM-dd")) @@ -131,7 +131,7 @@ class TradeStatLog { } - ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-01成交统计.xlsx", rets, "数据") + ExcelUtils.write("D:\\数据\\建行\\房产估值\\成交\\2022-02成交统计.xlsx", rets, "数据") println "表格生成完成" } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy index 8fc309322..be649cf9d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/ExcelDataTest.groovy @@ -1,25 +1,123 @@ package com.fangjia.script.ccbevaluation.data import com.fangjia.core.RegionData +import com.fangjia.script.ccbevaluation.CcbCommonUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.district.SeaweedDataMatchProcess import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.GeoHelper import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject +import com.mongodb.QueryOperators import groovy.sql.Sql DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") +DBCollection CITY_STAT = MongoUtils.getCollectionByName("fangjia_stat", "city_stat", "192.168.0.43") +DBCollection CITY_INFO = MongoUtils.getCollectionByName("fangjia_data_ana", "city_info", "192.168.0.43") +DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") def sqlInstance = Sql.newInstance("jdbc:oracle:thin:@192.168.0.236:1521:orcl", "GOOJIA", "GOOJIA", "oracle.jdbc.driver.OracleDriver") + +def ZS_MONGODB_URL = "mongodb://admin:G00MONGOADMIN@192.168.0.100:30009"; +DBCollection COMMUNITY_CONN = MongoUtils.getCollectionByName2("fangjia", "community", ZS_MONGODB_URL) + + + +ExcelUtils.multiThreadingModify("C:\\Users\\gaojuan\\Desktop\\city.xlsx",{d-> + if(d['B']){ + //def avg_price =CommonDatautils.queryCityPrice(d['B']) + //d['C'] = avg_price + +// def q = new BasicDBObject() +// q.put("area_name",new BasicDBObject( '$regex' , d['B'].replace("州","")+/.*$/)) +// def obj = COMMUNITY_CONN.findOne(q) +// if(obj){ +// d['D'] = obj.area_code +// println obj.area_name +" "+obj.area_code +// } + + //def obj = findDistrictPriceCount(d['B'], DateUtils.parseFromTightByPattern("2022-03-13","yyyy-MM-dd"), CITY_INFO) +// d['D'] = CommonDatautils.findAllDistrictCount("district", d['B']) +// d['E'] = findHouseCount(d['B'], 20210101, 20211231) +// +// def startDate = DateUtils.parseFromTightByPattern("2021-01-01", "yyyy-MM-dd") +// def endDate = DateUtils.parseFromTightByPattern("2021-12-31", "yyyy-MM-dd") +// def deals = findTradesDetailById(d['B'], startDate,endDate, DEAL_PRICE_COLL) +// d['F'] = deals.size() + + def cityCodeDataMap = CcbCommonUtils.findCityCodeValue() + d['G'] = findCcbDistCount(cityCodeDataMap.get(d['B']), sqlInstance) + } +},0) + +def findCcbDistCount(cityid, sqlInstance){ + + def sql = "select count(*) as count from bigdata_community_gj where cityId='${cityid}' and status=0 "; + def total = null + sqlInstance.eachRow(sql,{r-> + total = r.count + }) + return total +} + + +public static def findTradesDetailById(city, startDate, endDate, DEAL_PRICE_COLL){ + def q = new BasicDBObject() + q.put("city", city) + q.put("area", new BasicDBObject('$gte', 20)) + q.put("avg_price", new BasicDBObject('$gt', 0)) + q.put("trade_date", new BasicDBObject('$gte', startDate).append('$lte',endDate)) + def dbcursor = DEAL_PRICE_COLL.find(q) + def total = dbcursor.count() + def rets = new HashSet() + def lists = [] + for(def d : dbcursor){ + def key=d.city+"##"+d.match_region+"##"+d.district_name+"##"+d.area+"##"+d.avg_price+"##"+DateUtils.formatDateByPattern(d.trade_date,"yyyy-MM-dd") + if(!rets.contains(key)){ + rets << key + lists << d + } + } + return lists +} + + + +def findHouseCount(def city, def startDate, def endDate){ + def aliasStr = null; + def total = 0 + try{ + + String sql = "select count(*) as c from latest_seawater where cat='sell' and city='${city}' and p_date >= ${startDate} and p_date<${endDate} " + ElasticUtils.executeSqlQuery(sql,{ data-> + total = data.c + }) + }catch(Exception e){ + e.printStackTrace() + } + + return total +} + +def findDistrictPriceCount(def city, def weekend, def CITY_INFO){ + def q = new BasicDBObject() + q.put("city", city) + q.put("weekend", weekend) + def obj = CITY_INFO.findOne(city) + return obj +} + + + //ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\人工核实_8511.xlsx", { d -> // def newRegion = RegionData.formatRegion(d['A'], d['B']) // if(newRegion && !d['K']) { @@ -475,26 +573,26 @@ def city = " // //}) -ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d -> - if(d['A']){ - def dist = CommonDatautils.findDistrictById(d['A']) - if(dist){ - def mappingAlias = dist.mapping_alias - if(mappingAlias){ - if(mappingAlias.contains(d['B'])){ - mappingAlias.remove(d['B']) - DBObject udata = new BasicDBObject(); - udata.put("mapping_alias", mappingAlias); - udata.put("m_date", new Date()); - DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata)) - }else{ - println d['A']+"无映射别名" - } - }else{ - println d['A']+"映射别名为空" - } - }else{ - println d['A']+"没找到" - } - } -}) \ No newline at end of file +//ExcelUtils.modify("D:\\数据\\建行\\房产估值\\小区\\删除映射别名数据的副本.xlsx", { d -> +// if(d['A']){ +// def dist = CommonDatautils.findDistrictById(d['A']) +// if(dist){ +// def mappingAlias = dist.mapping_alias +// if(mappingAlias){ +// if(mappingAlias.contains(d['B'])){ +// mappingAlias.remove(d['B']) +// DBObject udata = new BasicDBObject(); +// udata.put("mapping_alias", mappingAlias); +// udata.put("m_date", new Date()); +// DbUtils.seaweed.update(new BasicDBObject("_id", d['A']), new BasicDBObject('$set', udata)) +// }else{ +// println d['A']+"无映射别名" +// } +// }else{ +// println d['A']+"映射别名为空" +// } +// }else{ +// println d['A']+"没找到" +// } +// } +//}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy index 5e58e7b38..2f5d02c2f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/MatchMapData.groovy @@ -30,15 +30,63 @@ class MatchMapData { static main(args) { def task = new MatchMapData() - task.run() +// task.run() // task.run1() // task.run2() // task.run3() // task.exportHouse() // task.addDistrictLevel() + task.run4() } + def run4() { + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\房产估值\\小区\\核实小区是否存在.xlsx", { d -> + def regionCode = null + if (d['A']) { + def region = d['B'] + def newRegion = RegionData.formatRegion(d['A'], region) + if (newRegion) { + regionCode = getRegionCode(newRegion) + def dist = CommonDatautils.findDistrict(d['A'], newRegion, d['C'], "district") + if (dist) { + d['F'] = "完全一致" + d['E'] = dist.name + d['D'] = dist.name_cat + d['G'] = dist.estate_type2 + } else { + dist = CommonDatautils.findSeaweedByMappingAlias(d['A'], newRegion, d['C'], "district") + if (dist) { + d['F'] = "映射别名" + d['E'] = dist.name + d['D'] = dist.name_cat + d['G'] = dist.estate_type2 + } else { + def rets = CommonDatautils.findDistrictByAlias(d['A'], newRegion, d['C'], "district") + if (rets) { + if(rets.size()>1){ + d['F'] = "别名-多个" + }else{ + d['F'] = "别名" + } + } + if (rets) { + dist = rets[0] + } + if (dist) { + d['F'] = "完全一致" + d['E'] = dist.name + d['D'] = dist.name_cat + d['G'] = dist.estate_type2 + } + } + } + } + } + + }, 0) + } + def run3(){ def files = ["合肥融合匹配_1215.xlsx"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TradeDistrictToDB.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TradeDistrictToDB.groovy new file mode 100644 index 000000000..274497135 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/data/TradeDistrictToDB.groovy @@ -0,0 +1,198 @@ +package com.fangjia.script.ccbevaluation.data + +import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.gj.OpenApiUtils +import com.fangjia.script.gj.district.FangjiaClientUtils +import com.fangjia.script.linxsh.util.AddressUtil +import com.fangjia.script.util.CosineSimilarAlgorithm +import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject + +class TradeDistrictToDB { + static main(args) { + def task = new TradeDistrictToDB(); +// task.run() + task.run1() + } + + def run1(){ + def index = 0 + def keys = [] + ExcelUtils.modify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d -> + if(d['D']){ + def key = d['A']+"##"+d['D']+"##"+d['C'] + if(!keys.contains(key)){ + def formatAddress = null + if(d['G']){ + formatAddress = AddressUtil.formatAddress(d['A'], d['F']) + } + + Long glat = null + Long glng = null + Long blat = null + Long blng = null + if(d['G']){ + glat = d['H'].toString().toDouble() * 10000000000L + glng = d['G'].toString().toDouble() * 10000000000L + def baidu = BaiDuApiUtils.geoconv(d['G'].toString().toDouble(), d['H'].toString().toDouble(), 3) + blng = baidu.x * 10000000000L + blat = baidu.y * 10000000000L + } + + def name = d['C'].replace("·","").replace("II","二") + def estateType2 = "普通住宅" + if(name.contains("别墅")){ + estateType2 = "别墅" + }else if(name.contains("商住")){ + estateType2 = "商住" + } + + keys << key + if(name){ + def dist = CommonDatautils.findDistrict(d['A'], d['D'], name, "district") + if(!dist){ + def data = new BasicDBObject(); + data.put("name_cat",0) + data.put("status",0) + data.put("cat","district") + data.put("c_date", new Date()) + data.put("m_date", new Date()) + data.put("city", d['A']) + data.put("region",d['D']) + data.put("name", name) + data.put("estate_type2",estateType2) + data.put("address", formatAddress) + if(glat){ + data.put("lat",glat) + data.put("lng",glng) + data.put("lat2",blat) + data.put("lng2",blng) + } + + def markMap = [:] + markMap.put("data_source", "机器入库") + data.put("mark_map",markMap) + index++ + DbUtils.seaweed.save(data); + FangjiaClientUtils.sendDistrictPriceCalculate(d['A'], d['D'], d['C'], 4); + println index+" "+data + + } + } + } + } + },0) + } + def run() { + def index = 0 + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行成交\\成交未入库记录(12月)-新增.xlsx", { d -> + def city = d['A'] + def region = d['B'] + def newRegion = RegionData.formatRegion(city, region) + if (newRegion) { + def map = findMapData(city, newRegion, d['C']) + if(map){ + d['D'] = newRegion + d['E'] = map.name + d['F'] = map.address + d['G'] = map.lng + d['H'] = map.lat + d['I'] = map.lng2 + d['J'] = map.lag2 + } + } + }, 0); + } + + def findMapData(def city, def region, name){ + def rets = OpenApiUtils.getGaoDeSearchLocation(city, region, name) + def map = findStandardObject2(city, region, name, null, rets); + if (!map || !map.lat) { + rets = OpenApiUtils.getBiDuLocation(city, region, name); + map = findStandardObject2(city, region, name, null, rets); + } + return map; + } + + + def findStandardObject2(def city, def region, def name, def address, def rets) { + if (rets) { + def bestObj = [:] + if (rets.size() >= 1) { + for (def mapObj : rets) { + def score = 0 + if((mapObj.adname && mapObj.adname.contains(region)) || (mapObj.region && mapObj.region.contains(region))){ + score = 0.2 + } + if(mapObj.name && name){ + def newName = mapObj.name; + def names = [] + if(mapObj.name && mapObj.name.indexOf("(")>-1){ + def name1 = mapObj.name.substring(0, mapObj.name.indexOf("(")) + names << name1 + def name2 = mapObj.name.substring(mapObj.name.indexOf("(")+1, mapObj.name.indexOf(")")) + names << name2 + }else{ + names << newName + } + def maxNameScore = 0 + for(def n : names){ + double score1 = CosineSimilarAlgorithm.getSimilarity(n, name); + if(score1 && score1>0){ + def nameScore = (score1 * 4)/10 + if(nameScore > maxNameScore){ + maxNameScore = nameScore + } + } + } + + if(maxNameScore){ + try{ + maxNameScore = DataHandleUtils.getDoubleRound(2, maxNameScore) + }catch(Exception e){ + maxNameScore = 0 + } + + } + score = maxNameScore + score; + } + + if(mapObj.address && address){ + double addressScore = CosineSimilarAlgorithm.getSimilarity(mapObj.address, address); + if(addressScore && addressScore>0){ + try{ + addressScore = DataHandleUtils.getDoubleRound(2, addressScore) + }catch(Exception e){ + addressScore = 0 + } + score = addressScore*4/10 + score + } + } + + if(!bestObj){ + mapObj.score = score + bestObj = mapObj + }else{ + if(score > bestObj.score){ + mapObj.score = score + bestObj = mapObj + } + } + + } + } + return bestObj + } + return null + + } + + + + +} + diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index dd91fbeaf..ef344bc84 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -218,7 +218,6 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name","玉泉路甲46号院") DBObject keys = new BasicDBObject(); if(fields){ @@ -918,6 +917,23 @@ class CommonDatautils { } return ret; } + + public static def findDistrictsByCityAndName(String city, String name, String cat){ + DBObject q = new BasicDBObject(); + q.put("city", city); + q.put("cat", cat); + q.put("name", name); + q.put("visible", 0); + + def list = [] + DSCursor cursor = getDataStoreConnection().find(q); + if(cursor){ + for(def obj : cursor){ + list << obj; + } + } + return list; + } public static def findDistrict(String city, String name, String cat){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdTest.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdTest.groovy new file mode 100644 index 000000000..22604b2ec --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/ThirdTest.groovy @@ -0,0 +1,278 @@ +package com.fangjia.script.fuzhou + +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil +import com.fangjia.script.thirdPriceCrawl.CrawalUtil +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import net.sf.json.JSONObject +import org.apache.commons.io.FileUtils +import com.fangjia.script.gj.district.SeaweedDataMatchProcess + +import java.util.concurrent.CountDownLatch + +class ThirdTest { + + DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") + def thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); + + static main(args) { + def task = new ThirdTest() +// task.run() +// task.run1() + task.run3() +// task.run4() + + } + + def run4(){ + ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d -> + if (!d['C']) { +// def name = d['A'].replace("(钢混)","").replace("(非钢混)","").replace("(混合)","") +// def lists = CommonDatautils.findDistrictsByCityAndName("福州",name, "district") +// if(lists){ +// for(def o : lists){ +//// if(d['B'] && d['B'].toString().contains(o.address)){ +// d['C'] = o.region +// d['D'] = o.name +// d['E'] = "名称去括号中的字" +//// } +// } +// } + def rMap = SeaweedDataMatchProcess.doMatch("福州", null, d['A'], d['B']); + if(rMap){ + d['C'] = rMap.region + d['D'] = rMap.name + d['E'] = "匹配接口" + } + + } + },0) + } + + + def run3() { +// ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d -> +// if(!d['F']){ +// def price = CommonDatautils.queryDistrictPrice("福州",d['C'], d['D']) +// d['F'] = price +// } +// },0) +// List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_DISTRICT_STAT_202202.csv")) +// def map1 = [:] +// for(def s : list1){ +// def row = s.split("@\\|@") +// def key = row[2]+"##"+row[3]+"##"+row[4] +// map1.put(key, row[5]) +// } + + List list1= FileUtils.readLines(new File("D:\\数据\\福州\\FJW_THIRD_PARTY_AVG_PRICE_202202.csv")) + def map1 = [:] + for(def str : list1){ + def s = str.split("@\\|@") + def key = s[2]+"##"+s[3]+"##"+s[4] + def priceMap = [:] + map1.put(key, priceMap) + + if(s.size()>=7 && s[6]){ + priceMap.put("zhugezhaofang", s[6]) + } + + if(s.size()>=8 && s[7]){ + priceMap.put("anjuke", s[7]) + } + + if(s.size()>=9 && s[8]){ + priceMap.put("soufang", s[8]) + } + + if(s.size()>=10 && s[9]){ + priceMap.put("lianjia", s[9]) + } + + if(s.size()>=11 && s[10]){ + priceMap.put("cityhouse", s[10]) + } + + if(s.size()>=12 && s[11]){ + priceMap.put("fangtan", s[11]) + } + + if(s.size()>=13 && s[12]){ + priceMap.put("fangdoudou", s[12]) + } + if(s.size()>=14 && s[13]){ + priceMap.put("maitian", s[13]) + } + + try{ + if(s.size()>=15 && s[14]){ + priceMap.put("58tc", s[14]) + } + }catch(Exception e){ + e.getMessage() + } + + } + + ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房价网数据-公司方提供.xlsx", { d -> +// if (!d['F']) { +// def lists = CommonDatautils.findDistrictsByCityAndName("福州", d['A'], "district") +// if(lists){ +// for(def o : lists){ +// if(o.address==d['B']){ +// d['C'] = o.region +// } +// } +// } +// } + def key = "福州##"+d['C']+"##"+d['D'] + def priceMap = map1.get(key) + if(priceMap){ + if(priceMap.zhugezhaofang){ + d['F'] = priceMap.zhugezhaofang + } + if(priceMap.anjuke){ + d['G'] = priceMap.anjuke + } + if(priceMap.soufang){ + d['H'] = priceMap.soufang + } + if(priceMap.lianjia){ + d['I'] = priceMap.lianjia + } + if(priceMap.cityhouse){ + d['J'] = priceMap.cityhouse + } + if(priceMap.fangtan){ + d['K'] = priceMap.fangtan + } + if(priceMap.fangdoudou){ + d['L'] = priceMap.fangdoudou + } + if(priceMap.maitian){ + d['M'] = priceMap.maitian + } + + if(priceMap.get("58tc")){ + d['N'] = priceMap.get("58tc") + } + } + }, 0) + } + + def run1(){ + ExcelUtils.modify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供-结果.xlsx",{d-> + if(d['B'] && !d['D']){ + if(d['B'].contains("号")){ +// def price = searchContent2("福州","https://fz.58.com", d['B'], 304) +// if(price){ +// d['C'] = price +// d['F'] = "58同城" +// } + def price = thirdPartyColl("福州", d['B']) + if(price){ + d['D'] = price; + d['F'] = "麦田" + } + + } + } + },0) + } + + def thirdPartyColl(def city, def address){ + def q = new BasicDBObject() + q.put("city", city) + q.put("address", address) + q.put("source","麦田") + q.put("s_date",202202) + def obj = thirdPartyColl.findOne(q) + if(obj){ + return obj.avg_price + } + return null; + } + + + //查看此关键字是否有数据 + def searchContent2(def city,def url,def keyword, def cityid){ + def newUrl = url+"/esf-ajax/community/pc/autocomplete/?city_id=${cityid}&kw="+URLEncoder.encode(keyword, "utf-8")+"&type=2" + def document = CrawalUtil.getDocumentByABuYun(newUrl, "访问过于频繁", "utf-8", null, true); + def price = null + if(document){ + println "【"+keyword +"】 查询" + def jsonObj = JSONObject.fromObject(document) + if(jsonObj && jsonObj.status=="ok"){ + if(jsonObj.data.size()>0){ + def datas = jsonObj.data + def d = datas[0] + price = d.price.toInteger() + } + }else{ + println "【"+keyword +"】 无结果" + } + } + return price; + } + + def run(){ + def map = findThirdPrice("福州") + ExcelUtils.multiThreadingModify("D:\\数据\\福州\\福州\\房屋评估数据-数据方公司提供.xlsx",{d-> + if(!d['F']){ + def lists = CommonDatautils.findDistrictsByCityAndName("福州",d['A'], "district") + if(!lists){ + lists = CommonDatautils.findDistrictByAlias("福州",null, d['A'], "district") + } + for(def o : lists){ +// if(o.address == d['B']){ + def priceMap = map.get(o.region+"##"+o.name) + d['C'] = priceMap.anjuke_esf_price + d['D'] = priceMap.maitian_esf_price + d['E'] = priceMap.beike_esf_price + d['F'] = o.region +// } + } + } + + },0) + } + + def findThirdPrice(city){ + def q = new BasicDBObject() + q.put("city", city) + q.put("s_date", new BasicDBObject('$gte', 202109)) + def dbcursor = AVERAGE_STATISTICS.find(q).sort(new BasicDBObject("s_date", -1)) + def map = [:] + for(def db : dbcursor) { + def key = db.region+"##"+db.name + def dataMap = map.get(key) + if(!dataMap){ + dataMap = [:] + map.put(key, dataMap) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.anjuke_esf_price && db.anjuke_esf_price>3000){ + dataMap.put("anjuke_esf_price", db.anjuke_esf_price) + } + if(!dataMap.containsKey("anjuke_esf_price") && db.get("58tc_esf_price") && db.get("58tc_esf_price")>3000){ + dataMap.put("anjuke_esf_price", db.get("58tc_esf_price")) + } + + if(!dataMap.containsKey("maitian_esf_price") && db.maitian_esf_price && db.maitian_esf_price>3000){ + dataMap.put("maitian_esf_price", db.maitian_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.beike_esf_price && db.beike_esf_price>3000){ + dataMap.put("beike_esf_price", db.beike_esf_price) + } + + if(!dataMap.containsKey("beike_esf_price") && db.get("lianjia_esf_price") && db.lianjia_esf_price>3000){ + dataMap.put("beike_esf_price", db.lianjia_esf_price) + } + } + return map; + } +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy index eeb71c20d..b49996401 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/YouDaDataParser.groovy @@ -37,7 +37,7 @@ class YouDaDataParser { public static final DBCollection youda_newhouse_coll = MongoUtils.getCollectionByName2("business", "res_newdealprice", offlineUrl) public static final DBCollection youda_esfhouse_coll = MongoUtils.getCollectionByName2("business", "res_esfdealprice", offlineUrl) - //线下:228 线上:45 + //线下:228 线上:45 测试 static def new_targetUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc" static def new_queryUrl = "http://192.168.0.228:9200/new_deal_price_lansi/doc/_search" @@ -48,7 +48,7 @@ class YouDaDataParser { def final AtomicInteger count_ = new AtomicInteger(0) - static def statDate = "2022-01-01" + static def statDate = "2022-02-01" static main(args) { def task = new YouDaDataParser(); // task.newHouseparse();//友达一手房存入mongo @@ -285,7 +285,7 @@ class YouDaDataParser { def esfParser(){ - def files = ["D:\\数据\\友达\\2022年2月\\查询1.DBF"] + def files = ["D:\\数据\\友达\\2022年3月\\查询1.DBF"] def datas = [] for(def fileName : files){ def rows = readDBF(fileName); @@ -490,8 +490,9 @@ class YouDaDataParser { if(s){ def doc = new groovy.json.JsonSlurper().parseText(s) def hits = doc.hits - if(hits){ + if(hits && hits.total instanceof Integer){ total = hits.total + } } return total @@ -565,7 +566,7 @@ class YouDaDataParser { if(obj.premisesName && obj.premisesName!="null"){ dbo.premisesName = obj.premisesName } - dbo.price = obj.price + dbo.price = obj.price.toInteger() dbo.region = obj.region dbo.resId = obj.ydId.toString() dbo.totalPrice = obj.totalPrice diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index 5f2bbe1b3..dadbf4926 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -36,7 +36,7 @@ public class BeiKeCrawlByChrome { def task = new BeiKeCrawlByChrome(); task.run() // task.run1() -// task.addDealRegion(); + task.addDealRegion(); } @@ -142,7 +142,7 @@ public class BeiKeCrawlByChrome { def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") - if(dealDate.getTime()>=endDate.getTime()){ + if(dealDate && dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) }else{ @@ -342,9 +342,9 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") -// if(e.text().contains("天津")){ + if(e.text().contains("天津")){ cityMap.put(e.text(), "https:"+href) -// } + } } } return cityMap diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy index d60d5f02c..c3249e561 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/GaoDeUtils.groovy @@ -514,8 +514,9 @@ class GaoDeUtils { // "2a1fb75990f2b39f6221e50892dd3988" // ] + //35925efd7333f1df52145e51b8e4e56a 15021384995 def KeyList = [ - "10bcb421d2ee4e51ed9268d6dca58ad1" + "10bcb421d2ee4e51ed9268d6dca58ad1"//17740818975 ] def count = RandomUtils.nextInt(KeyList.size()) return KeyList.get(count); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index 76719fb79..e10306939 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -187,7 +187,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-30", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl }else{ @@ -218,7 +218,7 @@ public class LianjiaCrawlByChrome { def durl = e.select(".title a").attr("href") def dealDate = e.select(".dealDate").text() dealDate = DateUtils.parseFromTightByPattern(dealDate, "yyyy.MM.dd") - def endDate = DateUtils.parseFromTightByPattern("2022-01-31", "yyyy-MM-dd") + def endDate = DateUtils.parseFromTightByPattern("2022-01-20", "yyyy-MM-dd") if(dealDate.getTime()>=endDate.getTime()){ detailUrls << durl parse(city, e) @@ -421,8 +421,7 @@ public class LianjiaCrawlByChrome { def href = e.attr("href") // if(!href.contains("fang.lianjia") && e.text().contains("福州")){ cityMap.put(e.text(), href) -// } - } + } } return cityMap } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy index 5443a7a6f..97dc2e689 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/DataStat.groovy @@ -40,10 +40,10 @@ class DataStat { def run(){ - def source = "58同城"// - def citys = findCrawlCity(source, 202201); + def source = "搜房"// + def citys = findCrawlCity(source, 202112); for(def city : citys){ - def count1 = findCount(source, city, 202201); + def count1 = findCount(source, city, 202112); def count2 = findCount(source, city, 202202); println city +" "+count1+" "+count2 // println count1 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy index 37d2cafbb..d7d81e90e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/Esf58Crawl.groovy @@ -36,8 +36,8 @@ class Esf58Crawl extends BaseBatch{ def static final DBCollection thirdPartyColl = CrawalDBUtil.getThirdPartyCol(); def static final distMap = [:] - def static useDistKeys = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","不","了","这","中","大","有","和","人","是","在","一","的","为","上","个","国","我","以","要","他","时","来","用","们","生","到","作","地","于","出","就","分","对","成","会","可","主","发","年","动","同","工","也","能","下","过","子","说","产","种","面","而","方","后","多","定","行","学","法","所","民","得","经","十","三","之","进","着","等","部","度","家","电","力","里","如","水","化","高","自","二","理","起","小","物","现","实","加","量","都","两","体","制","机","当","使","点","从","业","本","去","把","性","好","应","开","它","合","还","因","由","其","些","然","前","外","天","政","四","日","那","社","义","事","平","形","相","全","表","间","样","与","关","各","重","新","线","内","数","正","心","反","你","明","看","原","又","么","利","比","或","但","质","气","第","向","道","命","此","变","条","只","没","结","解","问","意","建","月","公","无","系","军","很","情","者","最","立","代","想","已","通","并","提","直","题","党","程","展","五","果","料","象","员","革","位","入","常","文","总","次","品","式","活","设","及","管","特","件","长","求","老","头","基","资","边","流","路","级","少","图","山","统","接","知","较","长","将","组","见","计","别","她","手","角","期","根","论","运","农","指","几","九","区","强","放","决","西","被","干","做","必","战","先","回","则","任","取","据","处","队","南","给","色","光","门","即","保","治","北","造","百","规","热","领","七","海","地","口","东","导","器","压","志","世","金","增","争","济","阶","油","思","术","极","交","受","联","什","认","六共","权","收","证","改","清","已","美","再","采","转","更","单","风","切","打","白","教","速","花","带","安","场","身","车","例","真","务","具","万","每","目","至","达","走","积","示","议","声","报","斗","完","类","八","离","华","名","确","才","科","张","信","马","节","话","米","整","空","元","况","今","集","温","传","土","许","步","群","广","石","记","需","段","研","界","拉","林","律","叫","且","究","观","越","织","装","影","算","低","持","音","众","书","布","复","容","儿","须","际","商","非","验","连","断","深","难","近","矿","千","周","委","素","技","备","半","办","青","省","列","习","响","约","支","般","史","感","劳","便","团","往","酸","历","市","克","何","除","消","构","府","称","太","准","精","值","号","率","族","维","划","选","标","写","存","候","毛","亲","快","效","斯","院","查","江","型","眼","王","按","格","养","易","置","派","层","片","始","却","专","状","育","厂","京","识","适","属","圆","包","火","住","调","满","县","局","照","参","红","细","引","听","该","铁","价","严","首","底","液","官","德","调","随","病","苏","失","尔","死","讲","配","女","黄","推","显","谈","罪","神","艺","呢","席","含","企","望","密","批","营","项","防","举","球","英","氧","势","告","李","台","落","木","帮","轮","破","亚","师","围","注","远","字","材","排","供","河","态","封","另","施","减","树","溶","怎","止","案","言","士","均","武","固","叶","鱼","波","视","仅","费","紧","爱","左","章","早","朝","害","续","轻","服","试","食","充","兵","源","判","护","司","足","某","练","差","致","板","田","降","黑","犯","负","击","范","继","兴","似","余","坚","曲","输","修","的","故","城","夫","够","送","笔","船","占","右","财","吃","富","春","职","觉","汉","画","功","巴","跟","虽","杂","飞","检","吸","助","升","阳","互","初","创","抗","考","投","坏","策","古","径","换","未","跑","留","钢","曾","端","责","站","简","述","钱","副","尽","帝","射","草","冲","承","独","令","限","阿","宣","环","双","请","超","微","让","控","州","良","轴","找","否","纪","益","依","优","顶","础","载","倒","房","突","坐","粉","敌","略","客","袁","冷","胜","绝","析","块","剂","测","丝","协","重","诉","念","陈","仍","罗","盐","友","洋","错","苦","夜","刑","移","频","逐","靠","混","母","短","皮","终","聚","汽","村","云","哪","既","距","卫","停","烈","央","察","烧","行","迅","境","若","印","洲","刻","括","激","孔","搞","甚","室","待","核","校","散","侵","吧","甲","游","久","菜","味","旧","模","湖","货","损","预","阻","毫","普","稳","乙","妈","植","息","扩","银","语","挥","酒","守","拿","序","纸","医","缺","雨","吗","针","刘","啊","急","唱","误","训","愿","审","附","获","茶","鲜","粮","斤","孩","脱","硫","肥","善","龙","演","父","渐","血","欢","械","掌","歌","沙","著","刚","攻","谓","盾","讨","晚","粒","乱","燃","矛","乎","杀","药","宁","鲁","贵","钟","煤","读","班","伯","香","介","迫","句","丰","培","握","兰","担","弦","蛋","沉","假","穿","执","答","乐","谁","顺","烟","缩","征","脸","喜","松","脚","困","异","免","背","星","福","买","染","井","概","慢","怕","磁","倍","祖","皇","促","静","补","评","翻","肉","践","尼","衣","宽","扬","棉","希","伤","操","垂","秋","宜","氢","套","笔","督","振","架","亮","末","宪","庆","编","牛","触","映","雷","销","诗","座","居","抓","裂","胞","呼","娘","景","威","绿","晶","厚","盟","衡","鸡","孙","延","危","胶","还","屋","乡","临","陆","顾","掉","呀","灯","岁","措","束","耐","剧","玉","赵","跳","哥","季","课","凯","胡","额","款","绍","卷","齐","伟","蒸","殖","永","宗","苗","川","炉","岩","弱","零","杨","奏","沿","露","杆","探","滑","镇","饭","浓","航","怀","赶","库","夺","伊","灵","税","了","途","灭","赛","归","召","鼓","播","盘","裁","险","康","唯","录","菌","纯","借","糖","盖","横","符","私","努","堂","域","枪","润","幅","哈","竟","熟","虫","泽","脑","壤","碳","欧","遍","侧","寨","敢","彻","虑","斜","薄","庭","都","纳","弹","饲","伸","折","麦","湿","暗","荷","瓦","塞","床","筑","恶","户","访","塔","奇","透","梁","刀","旋","迹","卡","氯","遇","份","毒","泥","退","洗","摆","灰","彩","卖","耗","夏","择","忙","铜","献","硬","予","繁","圈","雪","函","亦","抽","篇","阵","阴","丁","尺","追","堆","雄","迎","泛","爸","楼","避","谋","吨","野","猪","旗","累","偏","典","馆","索","秦","脂","潮","爷","豆","忽","托","惊","塑","遗","愈","朱","替","纤","粗","倾","尚","痛","楚","谢","奋","购","磨","君","池","旁","碎","骨","监","捕","弟","暴","割","贯","殊","释","词","亡","壁","顿","宝","午","尘","闻","揭","炮","残","冬","桥","妇","警","综","招","吴","付","浮","遭","徐","您","摇","谷","赞","箱","隔","订","男","吹","乐","园","纷","唐","败","宋","玻","巨","耕","坦","荣","闭","湾","键","凡","驻","锅","救","恩","剥","凝","碱","齿","截","炼","麻","纺","禁","废","盛","版","缓","净","睛","昌","婚","涉","筒","嘴","插","岸","朗","庄","街","藏","姑","贸","腐","奴","啦","惯","乘","伙","恢","匀","纱","扎","辩","耳","彪","臣","亿","璃","抵","脉","秀","萨","俄","网","舞","店","喷","纵","寸","汗","挂","洪","着","贺","闪","柬","爆","烯","津","稻","墙","软","勇","像","滚","厘","蒙","芳","肯","坡","柱","荡","腿","仪","旅","尾","轧","冰","贡","登","黎","削","钻","勒","逃","障","氨","郭","峰","币","港","伏","轨","亩","毕","擦","莫","刺","浪","秘","援","株","健","售","股","岛","甘","泡","睡","童","铸","汤","阀","休","汇","舍","牧","绕","炸","哲","磷","绩","朋","淡","尖","启","陷","柴","呈","徒","颜","泪","稍","忘","泵","蓝","拖","洞","授","镜","辛","壮","锋","贫","虚","弯","摩","泰","幼","廷","尊","窗","纲","弄","隶","疑","氏","宫","姐","震","瑞","怪","尤","琴","循","描","膜","违","夹","腰","缘","珠","穷","森","枝","竹","沟","催","绳","忆","邦","剩","幸","浆","栏","拥","牙","贮","礼","滤","钠","纹","弹","罢","拍","咱","喊","袖","埃","勤","罚","焦","潜","伍","墨","欲","缝","姓","刊","饱","仿","奖","铝","鬼","丽","跨","默","挖","链","扫","喝","袋","炭","污","幕","诸","弧","励","梅","奶","洁","灾","舟","鉴","苯","讼","抱","毁","率","懂","寒","智","埔","寄","届","跃","渡","挑","丹","艰","贝","碰","拔","爹","戴","码","梦","芽","熔","赤","渔","哭","敬","颗","奔","藏","铅","熟","仲","虎","稀","妹","乏","珍","申","桌","遵","允","隆","螺","仓","魏","锐","晓","氮","兼","隐","碍","赫","拨","忠","肃","缸","牵","抢","博","巧","壳","兄","杜","讯","诚","碧","祥","柯","页","巡","矩","悲","灌","龄","伦","票","寻","桂","铺","圣","恐","恰","郑","趣","抬","荒","腾","贴","柔","滴","猛","阔","辆","妻","填","撤","储","签","闹","扰","紫","砂","递","戏","吊","陶","伐","喂","疗","瓶","婆","抚","臂","摸","忍","虾","蜡","邻","胸","巩","挤","偶","弃","槽","劲","乳","邓","吉","仁","烂","砖","租","乌","舰","伴","瓜","浅","丙","暂","燥","橡","柳","迷","暖","牌","纤","秧","胆","详","簧","踏","瓷","谱","呆","宾","糊","洛","辉","愤","竞","隙","怒","粘","乃","绪","肩","籍","敏","涂","熙","皆","侦","悬","掘","享","纠","醒","狂","锁","淀","恨","牲","霸","爬","赏","逆","玩","陵","祝","秒","浙","貌","役","彼","悉","鸭","着","趋","凤","晨","畜","辈","秩","卵","署","梯","炎","滩","棋","驱","筛","峡","冒","啥","寿","译","浸","泉","帽","迟","硅","疆","贷","漏","稿","冠","嫩","胁","芯","牢","叛","蚀","奥","鸣","岭","羊","凭","串","塘","绘","酵","融","盆","锡","庙","筹","冻","辅","摄","袭","筋","拒","僚","旱","钾","鸟","漆","沈","眉","疏","添","棒","穗","硝","韩","逼","扭","侨","凉","挺","碗","栽","炒","杯","患","馏","劝","豪","辽","勃","鸿","旦","吏","拜","狗","埋","辊","掩","饮","搬","骂","辞","勾","扣","估","蒋","绒","雾","丈","朵","姆","拟","宇","辑","陕","雕","偿","蓄","崇","剪","倡","厅","咬","驶","薯","刷","斥","番","赋","奉","佛","浇","漫","曼","扇","钙","桃","扶","仔","返","俗","亏","腔","鞋","棱","覆","框","悄","叔","撞","骗","勘","旺","沸","孤","粘","吐","孟","渠","屈","疾","妙","惜","仰","狠","胀","谐","抛","霉","桑","岗","嘛","衰","盗","渗","脏","赖","涌","甜","曹","阅","肌","哩","厉","烃","纬","毅","昨","伪","症","煮","叹","钉","搭","茎","笼","酷","偷","弓","锥","恒","杰","坑","鼻","翼","纶","叙","狱","逮","罐","络","棚","抑","膨","蔬","寺","骤","穆","冶","枯","册","尸","凸","绅","坯","牺","焰","轰","欣","晋","瘦","御","锭","锦","丧","旬","锻","垄","搜","佛","扑","邀","亭","酯","迈","舒","脆","酶","闲","忧","酚","顽","羽","涨","卸","仗","陪","薄","辟","惩","杭","姚","肚","捉","飘","漂","昆","欺","吾","郎","烷","汁","呵","饰","萧","雅","邮","迁","燕","撒","姻","赴","宴","烦","削","债","帐","斑","铃","旨","醇","董","饼","雏","姿","拌","傅","腹","妥","揉","贤","拆","歪","葡","胺","丢","浩","徽","昂","垫","挡","览","贪","慰","缴","汪","慌","冯","诺","姜","谊","凶","劣","诬","耀","昏","躺","盈","骑","乔","溪","丛","卢","抹","易","闷","咨","刮","驾","缆","悟","摘","铒","掷","颇","幻","柄","惠","惨","佳","仇","腊","窝","涤","剑","瞧","堡","泼","葱","罩","霍","捞","胎","苍","滨","俩","捅","湘","砍","霞","邵","萄","疯","淮","遂","熊","粪","烘","宿","档","戈","驳","嫂","裕","徙","箭","捐","肠","撑","晒","辨","殿","莲","摊","搅","酱","屏","疫","哀","蔡","堵","沫","皱","畅","叠","阁","莱","敲","辖","钩","痕","坝","巷","饿","祸","丘","玄","溜","曰","逻","彭","尝","卿","妨","艇","吞","韦","怨","矮","歇","郊","禄","捻","漠","粹","颠","宏","冤","肪","饥","呵","仙","押","挨","醛","娃","拾","没","佩","勿","吓","讹","侯","恋","夕","锌","篡","戚","淋","蓬","岂","釉","兆","泊","魂","拘","亡","杠","摧","氟","颂","浑","凌","铀","诱","犁","谴","颁","舶","扯","嘉","萌","犹","滋","焊","舌","匹","媳","肺","掠","酿","烹","疲","驰","鸦","窄","辱","狭","朴","遣","菲","奸","韧","辣","拳","秆","卧","醉","竭","茅","墓","矣","哎","艳","敦","舆","缔","雇","尿","葬","履","契","禽","渣","衬","躲","赔","咸","溉","贼","醋","堤","抖","妃","裤","廉","晴","挽","掀","茫","丑","亥","拦","悠","阐","慧","佐","奇","竖","孝","柜","麟","绣","遥","逝","愁","肖","昭","芬","逢","窑","捷","圜","盲","闸","宙","辐","披","账","狼","幽","绸","蜂","慎","餐","酬","誓","惟","叉","弥","址","帜","芝","砌","唉","仆","涛","臭","翠","盒","劫","慨","炳","阖","寂","椒","倘","拓","畏","喉","巾","颈","垦","拚","兽","蔽","芦","乾","爽","窃","谭","挣","崩","模","褐","传","翅","儒","伞","晃","谬","胚","剖","凑","眠","浊","霜","礁","蔑","抄","闯","洒","碑","蓉","耶","猜","蹲","壶","唤","澳","锯","郡","玲","绵","纽","梳","掏","吁","锤","鼠","穴","椅","殷","遮","吵","萍","厌","畜","俱","夸","吕","囊","捧","雌","闽","饶","瞬","郁","哨","凿","朝","俺","浒","茂","肝","勋","盯","籽","耻","菊","滥","稼","戒","奈","帅","鞭","蚕","镁","询","跌","烤","坛","宅","笛","鄂","蛮","颤","棍","睁","鼎","岌","降","侍","藩","嚷","匪","岳","糟","缠","迪","泄","卑","氛","堪","萝","盛","碘","缚","悦","澄","甫","攀","屠","溢","拱","晰","携","朽","吟","菱","谦","凹","俊","芒","盼","婶","艘","酰","趁","唇","挫","羞","浴","疼","萎","肴","愚","肿","刨","绞","枢","嫁","慕","舱","铲","苹","豫","谕","迭","潘","顷","翁","榜","匠","欠","茬","畴","胃","沾","踪","弊","哼","鹏","歧","桐","沃","悼","惑","溃","蔗","荐","潭","孢","露","诊","庸","聪","嫌","厨","庞","祁","钳","肆","梭","赠","崖","篮","颖","甸","藻","捣","且","撕","诏","贞","赐","慈","炕","胖","兹","差","琼","锈","汛","卓","棵","馈","挠","灶","婴","蒂","肤","衫","沥","仑","勉","沪","逸","蜜","浦","嗓","晕","膏","祭","赢","艾","扮","鹅","怜","蒲","兔","孕","呖","蘖","挪","淑","谣","惧","廊","缅","俘","骄","膀","陡","宰","诞","峻","恼","腺","猎","涡","夷","愉","魔","铵","葛","贾","似","荫","哟","脊","钞","苛","锰","椭","镶","杏","溴","倚","滞","会","氓","捏","斩","傲","匆","僵","卤","烫","衍","榨","拢","裸","屑","咽","坊","舅","渴","翔","邪","拄","窖","猫","砌","钦","媒","脾","勺","柏","栅","噪","昼","耿","扁","辰","秤","得","贩","糕","梁","昙","衷","宦","扔","哇","诈","嘱","藤","卜","冈","悔","廓","皂","拐","氰","杉","玛","矢","寓","瓣","罕","垮","笋","淘","衔","称","恭","喇","帕","桉","秉","帘","铭","蛇","摔","斋","叭","帆","裸","俭","瘤","篷","砸","肢","辟","脖","瞪","暑","卜","竿","歼","笙","酮","蕴","哗","瞎","喀","刃","楔","喘","枚","嵌","挝","厢","粤","甩","拴","膝","恳","腕","娓","熄","锚","忌","愧","哦","荆","圃","骚","丸","蒜","毯","弗","俯","鹿","梢","屯","衙","轿","贱","垒","谅","踢","哑","滔","渥","饷","泳","棕","熬","搁","腈","梨","吻","樱","奠","捆","姨","柏","聘","惕","郓","绑","冀","裹","酥","寡","彦","稠","啡","钝","汝","擅","汰","鳙","埔","敞","嘿","逊","栋","谨","咖","鲤","雀","佣","庵","葫","贿","鳞","拼","搏","谎","塌","忉","腻","戊","怖","坟","禾","刹","嘻","桔","坎","拇","煽","狮","痒","曾","梗","寇","鹰","烛","哄","莽","雯","胳","龟","亟","糠","泌","坪","傻","什","喻","渊","蚌","跪","巷","涅","钊","譬","蕊","膛","侮","奕","枕","辫","况","扼","郝","寥","凄","厦","腥","钧","耦","蹄","戥","屁","诵","匈","桩","钓","涵","倦","袍","抒","屿","蹈","忿","敷","虹","聊","嗣","尉","灿","糙","蹬","嗯","姬","狡","笨","辜","僧","茨","讽","翰","枉","岐","枣","崭","焚","咕","猴","揽","涝","耍","趟","汹","咋","傍","镀","给","爵","虏","劈","璋","踩","瞅","迄","昔","汞","呱","诡","魄","祺","嘲","惶","赃","癌","咐","歉","扳","鄙","庐","聂","便","芡","躯","贬","煌","拧","隋","襄","淤","宠","炊","滇","謇","懒","栓","佑","憾","骆","裙","猖","兜","孵","痼","盥","曝","泣","絮","韵","眷","旷","噢","参","栖","盏","鳌","溅","煎","校","榴","暮","琪","淆","陛","巢","哒","吼","槐","唧","其","沛","乞","蜀","蜇","赚","捍","铰","幂","尧","咒","耽","叮","褂","焕","煞","雹","搓","釜","铬","拣","募","淹","瑰","鲢","茄","灼","邹","躬","觉","娇","焉","彰","鹤","琳","沦","畔","惹","庶","毙","皖","邢","禹","渍","绷","窜","翘","淫","箪","陌","膊","鞑","咳","玫","巫","拂","蕉","澜","赎","绥","锄","囱","赌","颊","缕","寅","躁","稚","庚","苟","氦","魁","珊","蜕","蛭","酌","逗","闺","蔓","撇","豌","朕","缉","襟","镍","桅","荧","侄","卒","佃","瞿","娶","饪","耸","乍","靶","痴","靖","扛","筐","韶","嚣","崔","蓿","岔","氘","娥","剿","霖","喃","搪","雍","裳","撰","豹","骏","慷"] - + //def static useDistKeys = ["a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z","不","了","这","中","大","有","和","人","是","在","一","的","为","上","个","国","我","以","要","他","时","来","用","们","生","到","作","地","于","出","就","分","对","成","会","可","主","发","年","动","同","工","也","能","下","过","子","说","产","种","面","而","方","后","多","定","行","学","法","所","民","得","经","十","三","之","进","着","等","部","度","家","电","力","里","如","水","化","高","自","二","理","起","小","物","现","实","加","量","都","两","体","制","机","当","使","点","从","业","本","去","把","性","好","应","开","它","合","还","因","由","其","些","然","前","外","天","政","四","日","那","社","义","事","平","形","相","全","表","间","样","与","关","各","重","新","线","内","数","正","心","反","你","明","看","原","又","么","利","比","或","但","质","气","第","向","道","命","此","变","条","只","没","结","解","问","意","建","月","公","无","系","军","很","情","者","最","立","代","想","已","通","并","提","直","题","党","程","展","五","果","料","象","员","革","位","入","常","文","总","次","品","式","活","设","及","管","特","件","长","求","老","头","基","资","边","流","路","级","少","图","山","统","接","知","较","长","将","组","见","计","别","她","手","角","期","根","论","运","农","指","几","九","区","强","放","决","西","被","干","做","必","战","先","回","则","任","取","据","处","队","南","给","色","光","门","即","保","治","北","造","百","规","热","领","七","海","地","口","东","导","器","压","志","世","金","增","争","济","阶","油","思","术","极","交","受","联","什","认","六共","权","收","证","改","清","已","美","再","采","转","更","单","风","切","打","白","教","速","花","带","安","场","身","车","例","真","务","具","万","每","目","至","达","走","积","示","议","声","报","斗","完","类","八","离","华","名","确","才","科","张","信","马","节","话","米","整","空","元","况","今","集","温","传","土","许","步","群","广","石","记","需","段","研","界","拉","林","律","叫","且","究","观","越","织","装","影","算","低","持","音","众","书","布","复","容","儿","须","际","商","非","验","连","断","深","难","近","矿","千","周","委","素","技","备","半","办","青","省","列","习","响","约","支","般","史","感","劳","便","团","往","酸","历","市","克","何","除","消","构","府","称","太","准","精","值","号","率","族","维","划","选","标","写","存","候","毛","亲","快","效","斯","院","查","江","型","眼","王","按","格","养","易","置","派","层","片","始","却","专","状","育","厂","京","识","适","属","圆","包","火","住","调","满","县","局","照","参","红","细","引","听","该","铁","价","严","首","底","液","官","德","调","随","病","苏","失","尔","死","讲","配","女","黄","推","显","谈","罪","神","艺","呢","席","含","企","望","密","批","营","项","防","举","球","英","氧","势","告","李","台","落","木","帮","轮","破","亚","师","围","注","远","字","材","排","供","河","态","封","另","施","减","树","溶","怎","止","案","言","士","均","武","固","叶","鱼","波","视","仅","费","紧","爱","左","章","早","朝","害","续","轻","服","试","食","充","兵","源","判","护","司","足","某","练","差","致","板","田","降","黑","犯","负","击","范","继","兴","似","余","坚","曲","输","修","的","故","城","夫","够","送","笔","船","占","右","财","吃","富","春","职","觉","汉","画","功","巴","跟","虽","杂","飞","检","吸","助","升","阳","互","初","创","抗","考","投","坏","策","古","径","换","未","跑","留","钢","曾","端","责","站","简","述","钱","副","尽","帝","射","草","冲","承","独","令","限","阿","宣","环","双","请","超","微","让","控","州","良","轴","找","否","纪","益","依","优","顶","础","载","倒","房","突","坐","粉","敌","略","客","袁","冷","胜","绝","析","块","剂","测","丝","协","重","诉","念","陈","仍","罗","盐","友","洋","错","苦","夜","刑","移","频","逐","靠","混","母","短","皮","终","聚","汽","村","云","哪","既","距","卫","停","烈","央","察","烧","行","迅","境","若","印","洲","刻","括","激","孔","搞","甚","室","待","核","校","散","侵","吧","甲","游","久","菜","味","旧","模","湖","货","损","预","阻","毫","普","稳","乙","妈","植","息","扩","银","语","挥","酒","守","拿","序","纸","医","缺","雨","吗","针","刘","啊","急","唱","误","训","愿","审","附","获","茶","鲜","粮","斤","孩","脱","硫","肥","善","龙","演","父","渐","血","欢","械","掌","歌","沙","著","刚","攻","谓","盾","讨","晚","粒","乱","燃","矛","乎","杀","药","宁","鲁","贵","钟","煤","读","班","伯","香","介","迫","句","丰","培","握","兰","担","弦","蛋","沉","假","穿","执","答","乐","谁","顺","烟","缩","征","脸","喜","松","脚","困","异","免","背","星","福","买","染","井","概","慢","怕","磁","倍","祖","皇","促","静","补","评","翻","肉","践","尼","衣","宽","扬","棉","希","伤","操","垂","秋","宜","氢","套","笔","督","振","架","亮","末","宪","庆","编","牛","触","映","雷","销","诗","座","居","抓","裂","胞","呼","娘","景","威","绿","晶","厚","盟","衡","鸡","孙","延","危","胶","还","屋","乡","临","陆","顾","掉","呀","灯","岁","措","束","耐","剧","玉","赵","跳","哥","季","课","凯","胡","额","款","绍","卷","齐","伟","蒸","殖","永","宗","苗","川","炉","岩","弱","零","杨","奏","沿","露","杆","探","滑","镇","饭","浓","航","怀","赶","库","夺","伊","灵","税","了","途","灭","赛","归","召","鼓","播","盘","裁","险","康","唯","录","菌","纯","借","糖","盖","横","符","私","努","堂","域","枪","润","幅","哈","竟","熟","虫","泽","脑","壤","碳","欧","遍","侧","寨","敢","彻","虑","斜","薄","庭","都","纳","弹","饲","伸","折","麦","湿","暗","荷","瓦","塞","床","筑","恶","户","访","塔","奇","透","梁","刀","旋","迹","卡","氯","遇","份","毒","泥","退","洗","摆","灰","彩","卖","耗","夏","择","忙","铜","献","硬","予","繁","圈","雪","函","亦","抽","篇","阵","阴","丁","尺","追","堆","雄","迎","泛","爸","楼","避","谋","吨","野","猪","旗","累","偏","典","馆","索","秦","脂","潮","爷","豆","忽","托","惊","塑","遗","愈","朱","替","纤","粗","倾","尚","痛","楚","谢","奋","购","磨","君","池","旁","碎","骨","监","捕","弟","暴","割","贯","殊","释","词","亡","壁","顿","宝","午","尘","闻","揭","炮","残","冬","桥","妇","警","综","招","吴","付","浮","遭","徐","您","摇","谷","赞","箱","隔","订","男","吹","乐","园","纷","唐","败","宋","玻","巨","耕","坦","荣","闭","湾","键","凡","驻","锅","救","恩","剥","凝","碱","齿","截","炼","麻","纺","禁","废","盛","版","缓","净","睛","昌","婚","涉","筒","嘴","插","岸","朗","庄","街","藏","姑","贸","腐","奴","啦","惯","乘","伙","恢","匀","纱","扎","辩","耳","彪","臣","亿","璃","抵","脉","秀","萨","俄","网","舞","店","喷","纵","寸","汗","挂","洪","着","贺","闪","柬","爆","烯","津","稻","墙","软","勇","像","滚","厘","蒙","芳","肯","坡","柱","荡","腿","仪","旅","尾","轧","冰","贡","登","黎","削","钻","勒","逃","障","氨","郭","峰","币","港","伏","轨","亩","毕","擦","莫","刺","浪","秘","援","株","健","售","股","岛","甘","泡","睡","童","铸","汤","阀","休","汇","舍","牧","绕","炸","哲","磷","绩","朋","淡","尖","启","陷","柴","呈","徒","颜","泪","稍","忘","泵","蓝","拖","洞","授","镜","辛","壮","锋","贫","虚","弯","摩","泰","幼","廷","尊","窗","纲","弄","隶","疑","氏","宫","姐","震","瑞","怪","尤","琴","循","描","膜","违","夹","腰","缘","珠","穷","森","枝","竹","沟","催","绳","忆","邦","剩","幸","浆","栏","拥","牙","贮","礼","滤","钠","纹","弹","罢","拍","咱","喊","袖","埃","勤","罚","焦","潜","伍","墨","欲","缝","姓","刊","饱","仿","奖","铝","鬼","丽","跨","默","挖","链","扫","喝","袋","炭","污","幕","诸","弧","励","梅","奶","洁","灾","舟","鉴","苯","讼","抱","毁","率","懂","寒","智","埔","寄","届","跃","渡","挑","丹","艰","贝","碰","拔","爹","戴","码","梦","芽","熔","赤","渔","哭","敬","颗","奔","藏","铅","熟","仲","虎","稀","妹","乏","珍","申","桌","遵","允","隆","螺","仓","魏","锐","晓","氮","兼","隐","碍","赫","拨","忠","肃","缸","牵","抢","博","巧","壳","兄","杜","讯","诚","碧","祥","柯","页","巡","矩","悲","灌","龄","伦","票","寻","桂","铺","圣","恐","恰","郑","趣","抬","荒","腾","贴","柔","滴","猛","阔","辆","妻","填","撤","储","签","闹","扰","紫","砂","递","戏","吊","陶","伐","喂","疗","瓶","婆","抚","臂","摸","忍","虾","蜡","邻","胸","巩","挤","偶","弃","槽","劲","乳","邓","吉","仁","烂","砖","租","乌","舰","伴","瓜","浅","丙","暂","燥","橡","柳","迷","暖","牌","纤","秧","胆","详","簧","踏","瓷","谱","呆","宾","糊","洛","辉","愤","竞","隙","怒","粘","乃","绪","肩","籍","敏","涂","熙","皆","侦","悬","掘","享","纠","醒","狂","锁","淀","恨","牲","霸","爬","赏","逆","玩","陵","祝","秒","浙","貌","役","彼","悉","鸭","着","趋","凤","晨","畜","辈","秩","卵","署","梯","炎","滩","棋","驱","筛","峡","冒","啥","寿","译","浸","泉","帽","迟","硅","疆","贷","漏","稿","冠","嫩","胁","芯","牢","叛","蚀","奥","鸣","岭","羊","凭","串","塘","绘","酵","融","盆","锡","庙","筹","冻","辅","摄","袭","筋","拒","僚","旱","钾","鸟","漆","沈","眉","疏","添","棒","穗","硝","韩","逼","扭","侨","凉","挺","碗","栽","炒","杯","患","馏","劝","豪","辽","勃","鸿","旦","吏","拜","狗","埋","辊","掩","饮","搬","骂","辞","勾","扣","估","蒋","绒","雾","丈","朵","姆","拟","宇","辑","陕","雕","偿","蓄","崇","剪","倡","厅","咬","驶","薯","刷","斥","番","赋","奉","佛","浇","漫","曼","扇","钙","桃","扶","仔","返","俗","亏","腔","鞋","棱","覆","框","悄","叔","撞","骗","勘","旺","沸","孤","粘","吐","孟","渠","屈","疾","妙","惜","仰","狠","胀","谐","抛","霉","桑","岗","嘛","衰","盗","渗","脏","赖","涌","甜","曹","阅","肌","哩","厉","烃","纬","毅","昨","伪","症","煮","叹","钉","搭","茎","笼","酷","偷","弓","锥","恒","杰","坑","鼻","翼","纶","叙","狱","逮","罐","络","棚","抑","膨","蔬","寺","骤","穆","冶","枯","册","尸","凸","绅","坯","牺","焰","轰","欣","晋","瘦","御","锭","锦","丧","旬","锻","垄","搜","佛","扑","邀","亭","酯","迈","舒","脆","酶","闲","忧","酚","顽","羽","涨","卸","仗","陪","薄","辟","惩","杭","姚","肚","捉","飘","漂","昆","欺","吾","郎","烷","汁","呵","饰","萧","雅","邮","迁","燕","撒","姻","赴","宴","烦","削","债","帐","斑","铃","旨","醇","董","饼","雏","姿","拌","傅","腹","妥","揉","贤","拆","歪","葡","胺","丢","浩","徽","昂","垫","挡","览","贪","慰","缴","汪","慌","冯","诺","姜","谊","凶","劣","诬","耀","昏","躺","盈","骑","乔","溪","丛","卢","抹","易","闷","咨","刮","驾","缆","悟","摘","铒","掷","颇","幻","柄","惠","惨","佳","仇","腊","窝","涤","剑","瞧","堡","泼","葱","罩","霍","捞","胎","苍","滨","俩","捅","湘","砍","霞","邵","萄","疯","淮","遂","熊","粪","烘","宿","档","戈","驳","嫂","裕","徙","箭","捐","肠","撑","晒","辨","殿","莲","摊","搅","酱","屏","疫","哀","蔡","堵","沫","皱","畅","叠","阁","莱","敲","辖","钩","痕","坝","巷","饿","祸","丘","玄","溜","曰","逻","彭","尝","卿","妨","艇","吞","韦","怨","矮","歇","郊","禄","捻","漠","粹","颠","宏","冤","肪","饥","呵","仙","押","挨","醛","娃","拾","没","佩","勿","吓","讹","侯","恋","夕","锌","篡","戚","淋","蓬","岂","釉","兆","泊","魂","拘","亡","杠","摧","氟","颂","浑","凌","铀","诱","犁","谴","颁","舶","扯","嘉","萌","犹","滋","焊","舌","匹","媳","肺","掠","酿","烹","疲","驰","鸦","窄","辱","狭","朴","遣","菲","奸","韧","辣","拳","秆","卧","醉","竭","茅","墓","矣","哎","艳","敦","舆","缔","雇","尿","葬","履","契","禽","渣","衬","躲","赔","咸","溉","贼","醋","堤","抖","妃","裤","廉","晴","挽","掀","茫","丑","亥","拦","悠","阐","慧","佐","奇","竖","孝","柜","麟","绣","遥","逝","愁","肖","昭","芬","逢","窑","捷","圜","盲","闸","宙","辐","披","账","狼","幽","绸","蜂","慎","餐","酬","誓","惟","叉","弥","址","帜","芝","砌","唉","仆","涛","臭","翠","盒","劫","慨","炳","阖","寂","椒","倘","拓","畏","喉","巾","颈","垦","拚","兽","蔽","芦","乾","爽","窃","谭","挣","崩","模","褐","传","翅","儒","伞","晃","谬","胚","剖","凑","眠","浊","霜","礁","蔑","抄","闯","洒","碑","蓉","耶","猜","蹲","壶","唤","澳","锯","郡","玲","绵","纽","梳","掏","吁","锤","鼠","穴","椅","殷","遮","吵","萍","厌","畜","俱","夸","吕","囊","捧","雌","闽","饶","瞬","郁","哨","凿","朝","俺","浒","茂","肝","勋","盯","籽","耻","菊","滥","稼","戒","奈","帅","鞭","蚕","镁","询","跌","烤","坛","宅","笛","鄂","蛮","颤","棍","睁","鼎","岌","降","侍","藩","嚷","匪","岳","糟","缠","迪","泄","卑","氛","堪","萝","盛","碘","缚","悦","澄","甫","攀","屠","溢","拱","晰","携","朽","吟","菱","谦","凹","俊","芒","盼","婶","艘","酰","趁","唇","挫","羞","浴","疼","萎","肴","愚","肿","刨","绞","枢","嫁","慕","舱","铲","苹","豫","谕","迭","潘","顷","翁","榜","匠","欠","茬","畴","胃","沾","踪","弊","哼","鹏","歧","桐","沃","悼","惑","溃","蔗","荐","潭","孢","露","诊","庸","聪","嫌","厨","庞","祁","钳","肆","梭","赠","崖","篮","颖","甸","藻","捣","且","撕","诏","贞","赐","慈","炕","胖","兹","差","琼","锈","汛","卓","棵","馈","挠","灶","婴","蒂","肤","衫","沥","仑","勉","沪","逸","蜜","浦","嗓","晕","膏","祭","赢","艾","扮","鹅","怜","蒲","兔","孕","呖","蘖","挪","淑","谣","惧","廊","缅","俘","骄","膀","陡","宰","诞","峻","恼","腺","猎","涡","夷","愉","魔","铵","葛","贾","似","荫","哟","脊","钞","苛","锰","椭","镶","杏","溴","倚","滞","会","氓","捏","斩","傲","匆","僵","卤","烫","衍","榨","拢","裸","屑","咽","坊","舅","渴","翔","邪","拄","窖","猫","砌","钦","媒","脾","勺","柏","栅","噪","昼","耿","扁","辰","秤","得","贩","糕","梁","昙","衷","宦","扔","哇","诈","嘱","藤","卜","冈","悔","廓","皂","拐","氰","杉","玛","矢","寓","瓣","罕","垮","笋","淘","衔","称","恭","喇","帕","桉","秉","帘","铭","蛇","摔","斋","叭","帆","裸","俭","瘤","篷","砸","肢","辟","脖","瞪","暑","卜","竿","歼","笙","酮","蕴","哗","瞎","喀","刃","楔","喘","枚","嵌","挝","厢","粤","甩","拴","膝","恳","腕","娓","熄","锚","忌","愧","哦","荆","圃","骚","丸","蒜","毯","弗","俯","鹿","梢","屯","衙","轿","贱","垒","谅","踢","哑","滔","渥","饷","泳","棕","熬","搁","腈","梨","吻","樱","奠","捆","姨","柏","聘","惕","郓","绑","冀","裹","酥","寡","彦","稠","啡","钝","汝","擅","汰","鳙","埔","敞","嘿","逊","栋","谨","咖","鲤","雀","佣","庵","葫","贿","鳞","拼","搏","谎","塌","忉","腻","戊","怖","坟","禾","刹","嘻","桔","坎","拇","煽","狮","痒","曾","梗","寇","鹰","烛","哄","莽","雯","胳","龟","亟","糠","泌","坪","傻","什","喻","渊","蚌","跪","巷","涅","钊","譬","蕊","膛","侮","奕","枕","辫","况","扼","郝","寥","凄","厦","腥","钧","耦","蹄","戥","屁","诵","匈","桩","钓","涵","倦","袍","抒","屿","蹈","忿","敷","虹","聊","嗣","尉","灿","糙","蹬","嗯","姬","狡","笨","辜","僧","茨","讽","翰","枉","岐","枣","崭","焚","咕","猴","揽","涝","耍","趟","汹","咋","傍","镀","给","爵","虏","劈","璋","踩","瞅","迄","昔","汞","呱","诡","魄","祺","嘲","惶","赃","癌","咐","歉","扳","鄙","庐","聂","便","芡","躯","贬","煌","拧","隋","襄","淤","宠","炊","滇","謇","懒","栓","佑","憾","骆","裙","猖","兜","孵","痼","盥","曝","泣","絮","韵","眷","旷","噢","参","栖","盏","鳌","溅","煎","校","榴","暮","琪","淆","陛","巢","哒","吼","槐","唧","其","沛","乞","蜀","蜇","赚","捍","铰","幂","尧","咒","耽","叮","褂","焕","煞","雹","搓","釜","铬","拣","募","淹","瑰","鲢","茄","灼","邹","躬","觉","娇","焉","彰","鹤","琳","沦","畔","惹","庶","毙","皖","邢","禹","渍","绷","窜","翘","淫","箪","陌","膊","鞑","咳","玫","巫","拂","蕉","澜","赎","绥","锄","囱","赌","颊","缕","寅","躁","稚","庚","苟","氦","魁","珊","蜕","蛭","酌","逗","闺","蔓","撇","豌","朕","缉","襟","镍","桅","荧","侄","卒","佃","瞿","娶","饪","耸","乍","靶","痴","靖","扛","筐","韶","嚣","崔","蓿","岔","氘","娥","剿","霖","喃","搪","雍","裳","撰","豹","骏","慷"] + def static useDistKeys = [] static def totalCount = new AtomicLong() def cityList = [ @@ -191,7 +191,7 @@ class Esf58Crawl extends BaseBatch{ void process(Object args) { //addContent("福州","https://fz.58.com/xiaoqu/", 70); - def citys = ["成都","重庆","杭州","深圳","郑州","长沙","哈尔滨","长春","石家庄","太原","惠州","厦门","洛阳","德阳","镇江","泰安","宜昌","银川","襄阳","江门","荆州","滨州","衡阳","遂宁","鞍山","赣州","聊城","枣庄","德州","宿迁","新乡","信阳","包头","乐山","桂林","眉山","驻马店","许昌","泸州","宜宾","衡水","营口","达州","广安","吉安","鄂尔多斯","孝感","宝鸡","湘潭","十堰","上饶","菏泽","漯河","邢台","承德","宜春","平顶山","赤峰","延边","韶关","内江","茂名","梅州","焦作","郴州","黄冈","红河","绥化","丹东","阳江","娄底","黄石","开封","邵阳","牡丹江","周口","自贡","咸宁","汉中","长治","大庆","抚顺","北海","衢州","呼伦贝尔","永州","濮阳","晋中","晋城","河源","渭南","巴中","运城","舟山","齐齐哈尔","曲靖","怀化","铁岭","宁德","佳木斯","朝阳","葫芦岛","资阳","鄂州","通辽","益阳","荆门","大理","辽阳","阳泉","榆林","梧州","黔南","乌兰察布","昌吉","巴音郭楞","玉溪"] + def citys = ["新余","松原","楚雄","吕梁","雅安","本溪","鹤壁","拉萨","黔东南","忻州","兴安盟","乌海","贵港","三门峡","仙桃","安康","白城","平凉","庆阳","潮州","白银","吴忠","防城港","延安","丽江","黔西南","喀什","钦州","鸡西","萍乡","石嘴山","鹰潭","保山","哈密","铜仁","武威","普洱","河池","酒泉","朔州","百色","鹤岗","张家界","定西","湘西","济源","昭通","中卫","张掖","克拉玛依","黑河","铜川","辽源","云浮","博尔塔拉","固原","白山","天门","潜江","贺州","临沧","伊春","临夏","石河子","来宾","双鸭山","嘉峪关","海东"] for(def city : citys){ def str = cityList.get(city) for(def i=0;i<1;i++){ @@ -255,21 +255,21 @@ class Esf58Crawl extends BaseBatch{ for(def k=0;k for(def city : citys){ From d49c9078453afd98f2464e2290230b19b24e2dd6 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Mon, 21 Mar 2022 22:04:32 +0800 Subject: [PATCH 206/212] =?UTF-8?q?=E5=BB=BA=E7=AD=91=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ccbevaluation/DistrictGuidePriceBatch.groovy | 2 +- .../script/ccbevaluation/GuiDingPriceBatch.groovy | 2 +- .../com/fangjia/script/gj/ccb/CcbCodeUtils.groovy | 13 ++++++++----- .../script/gj/paimai/BeiKeCrawlByChrome.groovy | 2 +- .../script/gj/paimai/LianjiaCrawlByChrome.groovy | 6 +++--- .../thirdPriceCrawl/SynchDistrictPrice.groovy | 2 +- 6 files changed, 15 insertions(+), 12 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy index f67de27ae..94741174c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/DistrictGuidePriceBatch.groovy @@ -25,7 +25,7 @@ class DistrictGuidePriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findGuidePrice(city, "2022-02-1", "2022-03-1") + def prices = CcbCommonUtils.findGuidePrice(city, "2022-03-1", "2022-04-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy index 2f6bdef80..412677ae5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/GuiDingPriceBatch.groovy @@ -44,7 +44,7 @@ class GuiDingPriceBatch extends BaseBatch{ def distCodeMap = CcbCommonUtils.findAllDistrictCodeValue(city) //添加自己库中的人工指导价 - def prices = CcbCommonUtils.findPersonPrice(city, "2022-02-1", "2022-03-1") + def prices = CcbCommonUtils.findPersonPrice(city, "2022-03-1", "2022-04-1") if(prices){ def distMap = getDistrictMap(city) def rets = createCityDatas(cityCodeDatas, distCodeMap, distMap, prices) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy index 9557d9c92..1dabf3ada 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/CcbCodeUtils.groovy @@ -62,12 +62,12 @@ class CcbCodeUtils { return null } -// 高层塔楼 0101 增加“超高层” 0108 -// 高层板楼 0102 增加“老洋房” 0109 +// 高层塔楼 0101 增加“超高层” 0109 +// 高层板楼 0102 增加“老洋房” 0113 // 小高层板楼 0103 增加“花园洋房” 0110 // 多层板楼 0104 增加“四合院” 0111 // 独栋别墅 0105 增加“老式别墅洋房” 0112 -// 联排别墅 0106 +// 联排别墅 0106 增加“双拼别墅” 0108 // 叠拼别墅 0107 // 其他建筑形式 0199 @@ -84,7 +84,10 @@ class CcbCodeUtils { types << "0110" } if(buildType.contains("老洋房")){ - types << "0109" + types << "0113" + } + if(buildType.contains("双拼")){ + types << "0108" } if(buildType.contains("叠拼")){ types << "0107" @@ -99,7 +102,7 @@ class CcbCodeUtils { types << "0104" } if(buildType.contains("超高层")){ - types << "0108" + types << "0109" } if(buildType.contains("小高层")){ types << "0103" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index dadbf4926..aa3cb1328 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("天津")){ + if(e.text().contains("南京")){ cityMap.put(e.text(), "https:"+href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index e10306939..fca2a5eff 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,9 +34,9 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); - task.run() +// task.run() // task.run1() -// task.addDealRegion(); + task.addDealRegion(); // task.standardRegion() } @@ -109,7 +109,7 @@ public class LianjiaCrawlByChrome { def standardRegion(){ def q = new BasicDBObject() - q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-01-01","yyyy-MM-dd"))) + q.put("trade_date", new BasicDBObject(QueryOperators.GTE, DateUtils.parseFromTightByPattern("2022-02-01","yyyy-MM-dd"))) q.put("match_id", new BasicDBObject(QueryOperators.EXISTS, false) ) def dbcursor = DEAL_PRICE_COLL.find(q) for(def d : dbcursor){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy index 3bcad4e62..1ce2d438f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/thirdPriceCrawl/SynchDistrictPrice.groovy @@ -26,7 +26,7 @@ class SynchDistrictPrice { def run(){ - def citys = ["乌鲁木齐"]//CommonUtils.findAllCity() + def citys = CommonUtils.findAllCity() sourceMap.each { k, v -> for(def city : citys){ From ac8068af470048c4a445be982062022bd876d35a Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Mar 2022 10:33:27 +0800 Subject: [PATCH 207/212] update --- .../script/ccbevaluation/ThirdPartyPriceBatch.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy index f2eebcbbc..421a0e4f7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ThirdPartyPriceBatch.groovy @@ -22,9 +22,9 @@ import java.util.concurrent.ConcurrentHashMap @BatchOwner(Watcher.GaoJuan) class ThirdPartyPriceBatch extends BaseBatch { - def sDate= 202201//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); - def startDate = 202111 - def endDate = 202201 + def sDate= 202202//Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); + def startDate = 202112 + def endDate = 202202 def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -6, "yyyyMM")); static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") From 26cb0cec5d4615cfdeff912bb8f1a9d95637c472 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 22 Mar 2022 16:19:41 +0800 Subject: [PATCH 208/212] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E5=AF=BC=E5=87=BA?= =?UTF-8?q?=E6=97=A5=E6=9C=9F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/ccbevaluation/ExportOracleData.groovy | 4 ++-- .../com/fangjia/script/ccbevaluation/TradeStatLog.groovy | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy index 20e49d87c..9d06d7cef 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ExportOracleData.groovy @@ -38,8 +38,8 @@ class ExportOracleData { exportCommunityBuileType() exportThirdPartyPrice2(lastMonth) - exportBigdataForsale("2022-01-19") - exportBigdataDeal("2022-01-01") + exportBigdataForsale("2022-02-19") + exportBigdataDeal("2022-02-01") //计算结果相关 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy index 778b9c5ea..afbaed5ee 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/TradeStatLog.groovy @@ -40,8 +40,8 @@ class TradeStatLog { static main(args) { def task = new TradeStatLog() - task.run()//统计最近3个月抓取的成交数据量 -// task.exportTrade()//导出未入库成交 +// task.run()//统计最近3个月抓取的成交数据量 + task.exportTrade()//导出未入库成交 // task.thirdPartyStat() // println Calendar.getInstance().get(Calendar.YEAR) // task.averageStat() @@ -481,8 +481,8 @@ class TradeStatLog { def headers = ["ID", "城市", "区域", "名称", "来源","价格", "面积","地址", "成交时间","过滤原因"] def rets = [] rets << headers - def startDate = "2022-01-01" - def endDate = "2022-02-01" + def startDate = "2022-02-01" + def endDate = "2022-03-01" for(def city : citys){ def trades = findTradesDetail(city, DateUtils.parseFromTightByPattern(startDate,"yyyy-MM-dd"), DateUtils.parseFromTightByPattern(endDate,"yyyy-MM-dd")) From a95dde3ce6b67514dfa163397425c5aad70332c5 Mon Sep 17 00:00:00 2001 From: Jian Shen Date: Fri, 25 Mar 2022 11:23:35 +0800 Subject: [PATCH 209/212] =?UTF-8?q?=E6=96=B0=E5=A2=9E=E4=BA=94=E7=BA=A7?= =?UTF-8?q?=E8=A1=8C=E6=94=BF=E5=8C=BA=E5=88=92V2=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../open/controllers/AddressController.java | 45 +++++++++++++++++++ .../src/main/resources/urlrewrite2.0.xml | 5 +++ 2 files changed, 50 insertions(+) diff --git a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java index 2fa5a4b4c..af890d880 100644 --- a/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java +++ b/projects/fangjia-open/src/main/java/com/fangjia/open/controllers/AddressController.java @@ -1797,6 +1797,51 @@ public class AddressController { return mav; } + /** + * 五级行政区划V2地址标准化接口 + * + * @param model + * @return + */ + public Object fiveDivisionStructure2Action(AddesssMatchModel model) { + + RestfulModelAndView mav = new RestfulModelAndView(); + if (StringUtils.isBlank(model.getAddress())) { + mav.setModel(BasicModel.buildBadRequestModel("地址不能为空")); + return mav; + } + + HttpRequest request = new HttpRequest(); + JSONObject json = null; + Map params = new HashMap(); + params.put("city", StringUtils.isNotBlank(model.getCity()) ? model.getCity() : CONFIG.getDefaultCity()); + params.put("address", model.getAddress()); + // 地址验证无效也输出标准化结果 + params.put("flag", "true"); + try{ + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8"); + } catch(Exception e){ + try { + json = request.getJsonContent("http://192.168.0.100:32080/FangJiaAddressService/fiveDivisionStructure/v2", params, "UTF-8"); + } catch(Exception ex){ + ex.printStackTrace(); + } + } + + if (json != null) { + int code = json.getInt("status"); + String msg = json.getString("msg"); + BasicModel basicModel = new BasicModel(code, msg); + if(json.containsKey("result")){ + basicModel.setResult(json.get("result")); + } + mav.setModel(basicModel); + } else { + mav.setModel(BasicModel.buildServerErrorModel().setMsg("服务升级中...")); + } + return mav; + } + /** * 详细地址标准化接口 * diff --git a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml index ba0a21b49..c17530d88 100644 --- a/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml +++ b/projects/fangjia-open/src/main/resources/urlrewrite2.0.xml @@ -38,5 +38,10 @@ /address/standard + + + /address/fiveDivisionStructure + + \ No newline at end of file From d816d79e25770208f8076c9eeb15d8b4d1ea0bcd Mon Sep 17 00:00:00 2001 From: gaojuan Date: Sun, 27 Mar 2022 12:23:26 +0800 Subject: [PATCH 210/212] =?UTF-8?q?=E6=88=90=E4=BA=A41->28?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../ronghe/ExportDistrictRongHe.groovy | 4 +- .../gj/paimai/BeiKeCrawlByChrome.groovy | 4 +- .../gj/paimai/LianjiaCrawlByChrome.groovy | 2 +- .../loudong/HouseBookChromeCrawl.groovy | 261 ++++++++++++++++++ 4 files changed, 266 insertions(+), 5 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy index fff8d13cf..d81974c8b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ccbevaluation/ronghe/ExportDistrictRongHe.groovy @@ -26,7 +26,7 @@ class ExportDistrictRongHe extends BaseBatch{ void process(Object args) { def curDate = DateUtils.formatDateByPattern(new Date()+1, "yyyyMMdd") //按天导(每天一个文件夹) - exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-2), "yyyy-MM-dd")) + exportBigdataDealFJW2(curDate, DateUtils.formatDateByPattern(DateUtils.getFirstDayOfMonth(new Date()+1,-1), "yyyy-MM-dd")) if(curDate.toString().endsWith("10")){ exportBigdataCommunityGJRH() @@ -471,7 +471,7 @@ class ExportDistrictRongHe extends BaseBatch{ } def allRows = [] - if(curDate.toString().endsWith("01")){ + if(curDate.toString().endsWith("28")){ def sql = "select CITYID, DEALID, CITYNAME, DISTID, DISTNAME, STREET, STREETNO, COMMID, COMMNAME, BUILDID, BUILDNO, BUILDNAME, UNITID, UNITNO, UNITNAME, HOUSEID, ADDRESS, BUILDYEAR, PROPTYPE, PROPYEARS, PERIODNUM, FLOORNUM, FLOORTYPE, FLOORCNT, BUILDTYPE, BUILDSTRU, BUILDAREA, LIVINGAREA, HOUSESTYLE, ROOMCNT, LIVINGROOMCNT, KITCHENCNT, BATHROOMCNT, HOUSEUSE, HOUSESTRU, HOUSEASPECT, FITMENT, EVALUATENO, DEALUNITPRICE, DEALTOTALPRICE, DECLAREPRICE, EVALUATEPRICE, CONTRACTNO, to_char(DEALDATE,'yyyy-mm-dd') as DEALDATE, DEALTYPE, PAYTYPE, DEALPERIOD, IFELEVATOR, COMPAREAGARDEN, COMPAREATERRACE, COMPAREABALCONY, COMPAREABAYWINDOW, COMPAREAATTIC, COMPAREAGARAGE, COMPAREABASEMENT, COMPAREAOTHER, DATASOURCE, AGENCYID, AGENCYNAME, REMARK, CREATETIME, UPDATETIME, to_char(CREATETIME,'yyyy-mm-dd hh24:mi:ss') as CREATETIME1,to_char(UPDATETIME,'yyyy-mm-dd hh24:mi:ss') as UPDATETIME1,SOURCECODE,WORKDATE from BIGDATA_DEAL_GJ where dealdate>=to_date('${startDateStr}','yyyy-MM-dd') and dealdate<=to_date('${endDateStr}','yyyy-MM-dd')";//CITYID='${cityCode}' and sqlInstance.eachRow(sql,{r-> def data = [] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy index aa3cb1328..6d75b18bd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/BeiKeCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class BeiKeCrawlByChrome { static main(args) { def task = new BeiKeCrawlByChrome(); - task.run() +// task.run() // task.run1() task.addDealRegion(); } @@ -342,7 +342,7 @@ public class BeiKeCrawlByChrome { if(cityEles){ for(def e : cityEles){ def href = e.attr("href") - if(e.text().contains("南京")){ + if(!e.text().contains("南京")){ cityMap.put(e.text(), "https:"+href) } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy index fca2a5eff..76d63a6a5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/paimai/LianjiaCrawlByChrome.groovy @@ -34,7 +34,7 @@ public class LianjiaCrawlByChrome { static main(args) { def task = new LianjiaCrawlByChrome(); -// task.run() + task.run() // task.run1() task.addDealRegion(); // task.standardRegion() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy new file mode 100644 index 000000000..31ab75ba6 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/loudong/HouseBookChromeCrawl.groovy @@ -0,0 +1,261 @@ +package com.fangjia.script.loudong + +import com.fangjia.script.crawl.JQueryUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import org.jsoup.Jsoup +import org.openqa.selenium.By +import org.openqa.selenium.WebDriver +import org.openqa.selenium.WebElement +import org.openqa.selenium.chrome.ChromeDriver +import org.openqa.selenium.interactions.Actions +import org.openqa.selenium.support.ui.ExpectedCondition +import org.openqa.selenium.support.ui.ExpectedConditions +import org.openqa.selenium.support.ui.WebDriverWait + +class HouseBookChromeCrawl { + + static DBCollection loudongConn = MongoUtils.getCollectionByName("fangjia_crawl", "loudong", "192.168.0.235", 27018) + static def city = "重庆" + static main(args) { + def task = new HouseBookChromeCrawl(); + task.run() + } + + def run(){ + System.setProperty("webdriver.chrome.driver", "C:\\Users\\gaojuan\\AppData\\Local\\Google\\Chrome\\Application\\chromedriver.exe"); + System.setProperty("http.nonProxyHosts", "localhost"); + + WebDriver driver = new ChromeDriver() + def listPageUrl = getIndexPage(driver) + def keywords = getListPage(driver,"http://www.house-book.com.cn/"+listPageUrl) + keywords.remove("城南家园") + keywords.remove("康居西城") + + println city +"共 "+keywords.size()+" 个小区" + getDistrictIndex(driver, keywords) + + } + + def getListPage(driver, url){ + driver.get(url) + + def cityDoc = Jsoup.parse(driver.getPageSource()) + def total = JQueryUtils.selectText(".result-count-number:eq(0)", cityDoc) + if(total){ + def page = getTotalPage(total, 6) + def keywords = getAllDist(driver, url, page); + return keywords + } + return null; + } + + def getDistrictIndex(driver, keywords){ + for(def word : keywords){ + def code = URLEncoder.encode(URLEncoder.encode(word, "utf-8"),"utf-8") + goToDetailIndex(driver, code) + + } + } + + def getAllDist(driver, url, page){ + def keywords = [] + for(def i=1;i<=page;i++){ + driver.get(url+"?selectPages="+i) + def doc = Jsoup.parse(driver.getPageSource()) + def eles = JQueryUtils.select(".title-area .title-text", doc) + for(def e : eles){ + def text = e.text() + if(text=="富州北郡"){ + println i + } + keywords << text + } + } + return keywords + } + + def getTotalPage(def total, def pageSize){ + def mode = total.toInteger()/pageSize + if(mode > mode.toInteger()){ + mode = mode.toInteger() +1 + } + return mode; + } + + + def goToDetailIndex(driver, code){ + def url = "http://www.house-book.com.cn/buildAction!pnameHome?pname="+code+"&price=&totalPrice=&bsqar=&htype=&btype=&sellKind=" + driver.get(url) + + WebDriverWait wait = new WebDriverWait(driver,20) + wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.className("main")))); + def docment = Jsoup.parse(driver.getPageSource()); + def dist_name = JQueryUtils.select(".info-pname", docment).text() + def addInfo = JQueryUtils.select(".addr-info", docment).text() + def region = null + if(addInfo.contains(":") && addInfo.contains("-")){ + region = addInfo.substring(addInfo.indexOf(":")+1, addInfo.indexOf("-")) + } + + def address = JQueryUtils.select(".addr-info span", docment).text() + +// wait.until(new ExpectedCondition(){ +// @Override +// public WebElement apply(WebDriver d){ +// return d.findElement(By.linkText("[更多]")) +// } +// }).click() + driver.findElement(By.linkText("[更多]")).click() +// Thread.sleep(10) +// wait.until(ExpectedConditions.visibilityOf(driver.findElement(By.id("shouUl")))); + def lis = checkElement(driver, "#shouUl .dongli"); + + def datas = [] + if(lis){ + for(def li : lis){ + def id = li.attr("id") + def houseNum = li.select(".title-span").text() + def map = [:] + map.id = id + map.city = city + map.region = region + map.district_name = dist_name + map.address = address + map.house_num = houseNum + map.num = li.select(".dong_ul2 li:contains(许可证号:)").text().replace("许可证号:","") + map.type = li.select(".dong_ul2 .title-span-hbType").text() + def floor_heigth = li.select(".dong_ul2 label:contains(总楼层:)").text().replace("总楼层:","").replace("层","") + if(floor_heigth){ + map.floor_heigth = floor_heigth + } + def room_count = li.select(".dong_ul2 label:contains(总户数:)").text() + if(room_count){ + room_count = room_count.substring(room_count.indexOf("总户数:")+4, room_count.lastIndexOf("户")) + if(room_count){ + map.household_count = room_count + } + } + datas << map + } + } + + println dist_name +" 共 "+datas.size()+" 栋" + for(def m : datas){ + try{ + driver.findElement(By.id(m.id)).click() + + def rows = parseDetail(driver) + println "栋divId: "+m.id +" 室:"+rows.size() + saveDate(m, rows) + }catch(Exception e){ + println dist_name+" 栋divId: "+m.id+" 找不到相应的div element" + } + + // driver.navigate().back(); + } + } + + def checkElement(driver, className){ + def loop = 0 + def stop = false + def lis = null + while (!stop){ + loop++ + def doc = Jsoup.parse(driver.getPageSource()); + lis = JQueryUtils.select(className, doc) + if(lis || loop>=10){ + stop = true; + }else{ + Thread.sleep(5000) + } + } + return lis + + } + + def parseDetail(driver){ + def trs = checkElement(driver, ".rooms tr"); + def rows = [] + for(def tr : trs){ + def tds = tr.select("td") + if(tds){ + def floor = null + for(def j=0;j + data.put(k,v) + } + map.each{k,v-> + data.put(k,v) + } + data.put("c_date", new Date()) + datas << data + } + } + if(datas){ + loudongConn.insert(datas) + } + } +} From 51b8bae1136d04a1d1bd9a1daead2476cb5f368d Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 29 Mar 2022 16:30:35 +0800 Subject: [PATCH 211/212] =?UTF-8?q?=E6=BD=9C=E5=8A=9B=E6=8C=87=E6=95=B0?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../script/commData/CommonDatautils.groovy | 14 +++++++++++++ .../script/gj/level/DistrictIndexBatch.groovy | 21 +++++++++++++++---- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index ef344bc84..239c9f83c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -3152,5 +3152,19 @@ class CommonDatautils { } return dbo; } + + static def findNewSqmetreRent(def city, def region, def name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("district_name", name) + + def dbcursor = SQMETRE_RENT_COLL.find(q).sort(new BasicDBObject().append("weekend", -1)).limit(1) + def avg_price = null + for(def d : dbcursor){ + avg_price = d.avg_price; + } + return avg_price; + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 8feaa621c..9113f0971 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -70,7 +70,6 @@ class DistrictIndexBatch extends BaseBatch{ def run(){ def cityCodeDatas = CcbCommonUtils.findCityCodeValue() def citys = CommonDatautils.findAllCity(); - citys.remove("上海") for(def city : citys){ def dists = CommonDatautils.findAllDistrictByCity(city, "district", ["city","alias","region","name", "finish_building_date","estate_charge","lat","lng","lng2","lat2"]) def code = cityCodeDatas.get(city) @@ -85,7 +84,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - calcute(city, dists, code, city_avg_total_price) + // calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) } @@ -119,7 +118,7 @@ class DistrictIndexBatch extends BaseBatch{ def total_price = getHousePrice(d, dists, priceMap, citycode); def avg_cost = getAvgCost(d, costDatas) - + def avg_price = priceMap.get(d.city+"##"+d.region+"##"+d.name) def map = [:] map.id = d._id.toString() map.city = d.city @@ -137,9 +136,10 @@ class DistrictIndexBatch extends BaseBatch{ //datas << [d.city, d.region, d.name, total_price_index, avg_charge_index, rank_index, avg_cost_index,potentiality] + map.sell_avg_price = avg_price def newUrl = targetUrl +"_doc/"+map.id httpSaveData(newUrl, map) - println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price + println "[${totalCount.get()}] " + d.name + " avg_cost: " + avg_cost + " avg_charge:" + avg_charge + " rank:" + rank + " total_price:" + total_price+"" } catch (Exception e) { e.printStackTrace() } finally { @@ -213,6 +213,9 @@ class DistrictIndexBatch extends BaseBatch{ def potentiality = total_price_index + avg_charge_index + rank_index + avg_cost_index + 4099.978248 potentiality = DataHandleUtils.getDoubleRound(6, potentiality) + + def year = CommonUtils.getBuildYear(d.finish_building_date) + def sqmetre_rent = CommonDatautils.findNewSqmetreRent(d.city,d.region,d.name) def map = [:] map.id = d._id.toString() map.city = d.city @@ -228,6 +231,15 @@ class DistrictIndexBatch extends BaseBatch{ map.avg_total_price = total_price_index map.month = "2022-03" map.m_date = DateUtil.formatDate(new Date(),"yyyy-MM-dd HH:mm:ss").replace(" ","T") + + map.finish_date = year + map.sell_avg_price = rMap.sell_avg_price + if(sqmetre_rent){ + map.sqmetre_rent = sqmetre_rent + } + if(rMap.sell_avg_price && sqmetre_rent){ + map.price_to_rent = (rMap.sell_avg_price/sqmetre_rent).toInteger() + } def newUrl = targetUrl +"_doc/"+map.id httpSaveData(newUrl, map) @@ -268,6 +280,7 @@ class DistrictIndexBatch extends BaseBatch{ rMap.put("price_rank", rs.price_rank) rMap.put("avg_cost", rs.avg_cost) rMap.put("avg_estate_charge", rs.avg_estate_charge) + rMap.put("sell_avg_price", rs.sell_avg_price) } return detailMap; } From bb96843c8d52110acc34e16cce48132e13e12677 Mon Sep 17 00:00:00 2001 From: gaojuan Date: Tue, 29 Mar 2022 16:32:59 +0800 Subject: [PATCH 212/212] =?UTF-8?q?=E5=8E=BB=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/script/gj/level/DistrictIndexBatch.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy index 9113f0971..4bd387c7a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/level/DistrictIndexBatch.groovy @@ -84,7 +84,7 @@ class DistrictIndexBatch extends BaseBatch{ city_avg_total_price = 531.2615913 } if(city_avg_total_price){ - // calcute(city, dists, code, city_avg_total_price) + calcute(city, dists, code, city_avg_total_price) calcuteIndex(city, dists, code, city_avg_total_price) }