diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy index 8914794b4..34a65ee51 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/commData/CommonDatautils.groovy @@ -218,7 +218,7 @@ class CommonDatautils { q.put("visible", 0); q.put("cat", cat); q.put("city", city); -// q.put("name", "工商大楼"); +// q.put("name","东寓新村") DBObject keys = new BasicDBObject(); if(fields){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy index 58fe4abed..fa59d2b45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy @@ -1,20 +1,37 @@ package com.fangjia.script.gj - +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.BasicDBObject +import com.mongodb.DBCollection +import com.mongodb.DBObject +import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.Connection import java.sql.ResultSet import java.sql.SQLException +import java.sql.Statement +import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.atomic.AtomicInteger +import java.util.regex.Matcher +import java.util.regex.Pattern class EsHouseStat { - public static EsDataSource dataSource = new EsDataSource() - private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.42", 52002) +// DBCollection LOG = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020) + +// public static JdbcDataSource dataSource = new JdbcDataSource() +// private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" + public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(50) static { dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -23,12 +40,88 @@ class EsHouseStat { dataSource.setProperties(connectionProperties) } + public static final def lineUrl = "mongodb://developer:G00IDCDEV12385@192.168.0.43:28017"; + public static final DBCollection DEAL_PRICE_COLL = MongoUtils.getCollectionByName2("fangjia", "deal_price", lineUrl) + public static void main(String[] args){ def task = new EsHouseStat(); //task.run(); -// task.testRun("常州") - task.exportOpenLog(); +// task.testRun("深圳") +// task.exportOpenLog(); +// task.exportMongoLog() // task.exportPoi(); +// task.cityStat() +// task.stat(); +// task.matchStat(); +// task.statNoHouseCalType() +// task.statCityHouse() + +// task.szRentHouseStat(); +// task.addPrice(); + + task.statNoHouseCCBData(); + } + + def addPrice(){ + def datas = [] + def data1 = ExcelUtils.read("D:\\数据\\建行银行\\穆良书\\北京成都.xlsx",0,1) + def data2 = ExcelUtils.read("D:\\数据\\建行银行\\穆良书\\大连杭州厦门.xlsx",0,1) + def data3 = ExcelUtils.read("D:\\数据\\建行银行\\穆良书\\上海.xlsx",0,1) + def data4 = ExcelUtils.read("D:\\数据\\建行银行\\穆良书\\深圳广州.xlsx",0,1) + def data5 = ExcelUtils.read("D:\\数据\\建行银行\\穆良书\\重庆苏州.xlsx",0,1) + datas.addAll(data1) + datas.addAll(data2) + datas.addAll(data3) + datas.addAll(data4) + datas.addAll(data5) + + println "data1共:"+data1.size() + println "data2共:"+data2.size() + println "data3共:"+data3.size() + println "data4共:"+data4.size() + println "data5共:"+data5.size() + println "共:"+datas.size() + + final CountDownLatch latch = new CountDownLatch(datas.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + +// for(def data : data1){ +// def dbo = CommonDatautils.queryDistrictPrice(data[0], data[1], data[2], DateUtils.parseFromTightYYYYMMDD("2021-02-07")) +// data << dbo.avg_price +// } + + + for(def i=0;i0){ + results << dist + } + } + ExcelUtils.write("D:\\数据\\建行银行\\穆良书\\建行10个城市价格数据2.xlsx",results,"数据") + } def run(){ @@ -61,6 +154,189 @@ class EsHouseStat { } + def szRentHouseStat(){ + def citys = ["上海","北京","广州","深圳","杭州","南京","天津","成都","西安","沈阳","重庆","苏州","武汉","长沙","青岛","厦门","宁波","大连","无锡"] +// def citys = ["长沙"] + def rets = [] + rets << ["城市","小区数量","2020-08-01","2020-09-01","2020-10-01","2020-11-01","2020-12-01","2021-01-01"] + for(def city : citys){ + def month0 = findRentHouse(city, 20200801, 20210201) +// def month1 = findRentHouse(city, 20200801, 20200901) +// def month2 = findRentHouse(city, 20200901, 20201001) +// def month3 = findRentHouse(city, 20201001, 20201101) +// def month4 = findRentHouse(city, 20201101, 20201201) +// def month5 = findRentHouse(city, 20201201, 20210101) +// def month6 = findRentHouse(city, 20210101, 20210201) + + def districts = CommonDatautils.findAllDistrictByCity(city, "district", ["city","region", "name","alias"]) + def map = findMatchDistrictCount(city, districts); + +// def matchCount1 = 0; +// def matchCount2 = 0; +// def matchCount3 = 0; +// def matchCount4 = 0; +// def matchCount5 = 0; +// def matchCount6 = 0; +// for(def dist : districts){ +// def nameQuery = getAllName(dist); +// def month11 = findMatchHouse(city, dist.region, nameQuery, 20200801, 20200901) +// if(month11>0){ +// matchCount1 ++ +// } +// def month21 = findMatchHouse(city, dist.region, nameQuery, 20200901, 20201001) +// if(month21>0){ +// matchCount2 ++ +// } +// def month31 = findMatchHouse(city, dist.region, nameQuery, 20201001, 20201101) +// if(month31>0){ +// matchCount3 ++ +// } +// def month41 = findMatchHouse(city, dist.region, nameQuery, 20201101, 20201201) +// if(month41>0){ +// matchCount4 ++ +// } +// def month51= findMatchHouse(city, dist.region, nameQuery, 20201201, 20210101) +// if(month51>0){ +// matchCount5 ++ +// } +// def month61 = findMatchHouse(city, dist.region, nameQuery, 20210101, 20210201) +// if(month61>0){ +// matchCount6++ +// } + +// rets << [city, districts.size(), month1+"(${map.matchCount1})",month2+"(${map.matchCount2})",month3+"(${map.matchCount3})",month4+"(${map.matchCount4})",month5+"(${map.matchCount5})",month6+"(${map.matchCount6})" ] + rets << [city, districts.size(), month0] + println city +" 完成" + } + + ExcelUtils.write("d:\\数据\\租房数据统计1.xlsx",rets, "租房统计") + } + + + + def findMatchDistrictCount(def city, def districts){ + def rets = [] + final CountDownLatch latch = new CountDownLatch(districts.size()) + def final AtomicInteger count_ = new AtomicInteger(0) + def matchCount0 = 0; + def matchCount1 = 0; + def matchCount2 = 0; + def matchCount3 = 0; + def matchCount4 = 0; + def matchCount5 = 0; + def matchCount6 = 0; + for(def i=0;i0){ + matchCount0 ++ + } +// def month11 = findMatchHouse(city, dist.region, nameQuery, 20200801, 20200901) +// if(month11>0){ +// matchCount1 ++ +// } +// def month21 = findMatchHouse(city, dist.region, nameQuery, 20200901, 20201001) +// if(month21>0){ +// matchCount2 ++ +// } +// def month31 = findMatchHouse(city, dist.region, nameQuery, 20201001, 20201101) +// if(month31>0){ +// matchCount3 ++ +// } +// def month41 = findMatchHouse(city, dist.region, nameQuery, 20201101, 20201201) +// if(month41>0){ +// matchCount4 ++ +// } +// def month51= findMatchHouse(city, dist.region, nameQuery, 20201201, 20210101) +// if(month51>0){ +// matchCount5 ++ +// } +// def month61 = findMatchHouse(city, dist.region, nameQuery, 20210101, 20210201) +// if(month61>0){ +// matchCount6++ +// } + + }catch (Exception e){ + + }finally { + latch.countDown() + } + + } + } + ) + } + latch.await() + + def map = [:] + map.matchCount0 = matchCount0 +// map.matchCount1 = matchCount1 +// map.matchCount2 = matchCount2 +// map.matchCount3 = matchCount3 +// map.matchCount4 = matchCount4 +// map.matchCount5 = matchCount5 +// map.matchCount6 = matchCount6 + return map; + } + + def findMatchHouse(def city, def region, def nameQuery, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and region = '"+region+"'") + sqlBuffer.append(" and cat = 'rent'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + sqlBuffer.append(" and (" + nameQuery + ")"); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def total = 0; + while (rs.next()){ + total = rs.getInt("count") + } + return total; + } + + def getAllName(dist){ + Set aliasNames = new LinkedHashSet(); + aliasNames.add(dist.name); + if (dist != null && dist.alias != null) { + aliasNames.addAll(dist.alias); + } + + List namePars = new ArrayList(); + if(aliasNames!=null && aliasNames.size()>0) { + for(String name : aliasNames){ + if(StringUtils.isNotBlank(name) && name.trim().length() > 1){ + namePars.add("district_name ='" + name + "'"); + } + } + } + + if(namePars.size() > 0){ + return StringUtils.join(namePars, " OR ") + } + return null; + } + + def findRentHouse(def city, def startDate, def endDate){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'rent'") + sqlBuffer.append(" and p_date >="+startDate).append(" and p_date <"+endDate); + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def total = 0; + while (rs.next()){ + total = rs.getInt("count") + } + return total; + } + def findHouse(def city, def startDate, def endDate){ final StringBuffer sqlBuffer = new StringBuffer() sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") @@ -116,46 +392,344 @@ class EsHouseStat { def findLogs3(){ - //QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007736')).must(QueryBuilders.termQuery("request_uri", '/property/detail')) - QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007713')).must(QueryBuilders.termQuery("request_uri", '/property/evaluate')).must(QueryBuilders.termQuery("code", 200)) - def rets = [] - def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); - def curDate = DateUtils.parseFromTightByPattern("2020-09-01", "yyyy-MM-dd") - if(houses){ - for (int k=0;k= curDate.getTime()){// && - def ret = [] - def paramMap = map.parameter_map - def param_id = map.id - ret << param_id - ret << map.company_name - ret << map.ip - ret << map.request_uri - ret << map.c_date.toString().substring(0, map.c_date.indexOf("T")) - ret << map.data_status; - ret << map.code - ret << map.msg + //QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007736')).must(QueryBuilders.termQuery("request_uri", '/property/detail')) + // QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007721')).must(QueryBuilders.termQuery("request_uri", '/property/evaluate')).must(QueryBuilders.termQuery("code", 200)) + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007751')).must(QueryBuilders.termQuery("request_uri", '/property/evaluateForTest')).must(QueryBuilders.termQuery("parameter_map.city", '上海')) + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + def curDate = DateUtils.parseFromTightByPattern("2020-07-01", "yyyy-MM-dd") + if(houses){ + for (int k=0;k= curDate.getTime() + def ret = [] + def paramMap = map.parameter_map + def param_id = map.id + ret << param_id + ret << map.company_name + ret << map.ip + ret << map.request_uri + ret << map.c_date.toString().substring(0, map.c_date.indexOf("T")) + ret << map.data_status; + ret << map.code + ret << map.msg - def paramStr = "" - paramMap.each{key,val-> - if(key!="biz_code"){ - paramStr = paramStr+" "+key+":"+val - } + def paramStr = "" + paramMap.each{key,val-> + if(key!="biz_code"){ + paramStr = paramStr+" "+key+":"+val } - ret << paramStr.toString() - - def resMap = map.res_map - ret << resMap.result.toString() - rets << ret } + ret << paramStr.toString() + def resMap = map.res_map + ret << resMap.result.toString() + rets << ret + } + + } + } + return rets; + } + + + def findLogs4(def city){ + + def time = DateUtils.parseFromTightYYYYMMDD("2021-01-19") + QueryBuilder date_range_cdate = QueryBuilders.rangeQuery("c_date").from(time.getTime()) + + QueryBuilder queryBuilder = QueryBuilders.boolQuery().must(QueryBuilders.termQuery("user_id", '1007751')).must(QueryBuilders.termQuery("request_uri", '/property/evaluateForTest')).must(QueryBuilders.termQuery("parameter_map.city", city)).must(date_range_cdate) + def rets = [] + def houses = ElasticUtils.executeSimpleQuery("192.168.0.206", "open_uri_log", queryBuilder); + //"城市","楼盘名称","地址", + rets << ["城市","面积","室","厅","卫","楼高","物业类型","房龄","装修","朝向","估价时间","结果信息","城市","区域","楼盘名称","总价","单价","计算方式","房源列表量","建筑年代","物业类型","距离","近半年涨跌幅","同区域20%小区价位","同区域80%小区价位","附近小区量","经纬度"] + if(houses){ + for (int k=0;k= curDate.getTime() +// if(date.contains("2021-01-19")){ + + def ret = [] + def paramMap = map.parameter_map + //城市 楼盘名称 地址 面积 室 厅 卫 楼高 朝向 装修 房龄 估价时间 物业类型 + ret << cDate +// ret << paramMap.name +// ret << paramMap.address + ret << paramMap.size + ret << paramMap.room + ret << paramMap.hall + ret << paramMap.toilet + ret << paramMap.maxFloor + ret << paramMap.propertyType + ret << paramMap.age + ret << paramMap.decoration + ret << paramMap.direction + ret << paramMap.calDate + + def resMap = map.res_map.result +// println resMap +// if(!resMap){ +// println "sss" +// } + if(!resMap){ + ret << map.res_map.msg + }else{ + if(resMap.houses){ + //城市 区域 楼盘名称 总价 单价 计算方式 房源列表 + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << resMap.houses.size() + }else{ + //城市 区域 楼盘名称 总价 单价 计算方式 模型特征(经纬度,建筑年代,附近小区,距离,近半年涨跌幅,同区域20%,80%价位) + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << 0 + ret << resMap.buildYear + ret << resMap.modelFeatures.propertyType + ret << resMap.modelFeatures.distance + ret << resMap.modelFeatures.halfYearMarkup + ret << resMap.modelFeatures.region20PercentPrice + ret << resMap.modelFeatures.region80PercentPrice + ret << resMap.modelFeatures.nearCommunity.size() + ret << resMap.modelFeatures.lat+" "+resMap.modelFeatures.lng + + rets << ret + + } + } + +// }else{ +// println map.c_date +// } + } + } + return rets; + } + + def findTotalCount(def city) { + + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + def count = LOG.count(q) + return count; + + } + + def findLogs6(def city, def vals) { + + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + q.put("msg", new BasicDBObject('$in', vals)) + def count = LOG.count(q) + return count; + + } + + def findHouseType(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q, keys) + def count = 0 + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + if(resMap.calcType==value){ + count ++; } } - return rets; + } + return count; + } + + + def findHousePriceType(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def count = 0 + BigDecimal totalPrice = 0 + BigDecimal totalArea = 0 + BigDecimal totalAvgPrice = 0 + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + def price = resMap.totalPrice + def area = d.parameter_map.size + if(resMap.calcType==value){ + count ++; + totalPrice = totalPrice + price + totalArea = totalArea + area.toDouble() + } + } + } + + def map = [:] + map.totalPrice = totalPrice + map.avgPrice = (totalPrice/totalArea).toInteger() + return map; + + } + + + def findLogs5(def city){ + + def q = new BasicDBObject() + q.put("user_id","1007751") + q.put("parameter_map.city",city) + q.put("request_uri" ,"/property/evaluateForTest") +// q.put("c_date" ,new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-19"))) + def rets = [] +// rets << ["城市","楼盘名称","地址","面积","室","厅","卫","楼高","物业类型","房龄","装修","朝向","估价时间","结果信息","城市","区域","楼盘名称","总价","单价","计算方式","房源列表量","建筑年代","物业类型","近半年涨跌幅","同区域20%小区价位","同区域80%小区价位","附近小区量","经纬度"] + def houses = LOG.find(q) +// def count = LOG.count(q) + println LOG.count(q) + if(houses){ + def i = 0 + for(def map : houses){ + i++ + println i +" "+map +// def map = houses[k]; +// def date = map.c_date +// def cDate = DateUtils.parseFromTightByPattern(date.toString().substring(0, date.toString().indexOf("T")), "yyyy-MM-dd") +// if(map.code == 200){// && && cDate.getTime()>= curDate.getTime() + if(map.msg=="该小区不存在" || map.msg=="暂时不支持该小区的估价"){ + continue + } + def ret = [] + def paramMap = map.parameter_map + //城市 楼盘名称 地址 面积 室 厅 卫 楼高 朝向 装修 房龄 估价时间 物业类型 + ret << paramMap.city + ret << paramMap.name + ret << paramMap.address + ret << paramMap.size + ret << paramMap.room + ret << paramMap.hall + ret << paramMap.toilet + ret << paramMap.maxFloor + ret << paramMap.propertyType + ret << paramMap.age + ret << paramMap.decoration + ret << paramMap.direction + ret << paramMap.calDate + + def resMap = map.res_map.result + println resMap + + if(!resMap){ + ret << map.res_map.msg + }else{ + if(resMap.houses){ + //城市 区域 楼盘名称 总价 单价 计算方式 房源列表 + ret << "" + ret << resMap.city + ret << handleIllegalCharacter(resMap.district) + ret << handleIllegalCharacter(resMap.name) + ret << resMap.totalPrice + ret << resMap.unitPrice + ret << resMap.calcType + ret << resMap.houses.size() + + }else{ + //城市 区域 楼盘名称 总价 单价 计算方式 模型特征(经纬度,建筑年代,附近小区,距离,近半年涨跌幅,同区域20%,80%价位) + ret << "" + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + ret << 0 + ret << resMap.modelFeatures.buildYear + ret << resMap.modelFeatures.propertyType + ret << resMap.modelFeatures.halfYearMarkup + ret << resMap.modelFeatures.region20PercentPrice + ret << resMap.modelFeatures.region80PercentPrice + ret << resMap.modelFeatures.nearCommunity.size() + ret << resMap.modelFeatures.lat+" "+resMap.modelFeatures.lng + + } + } + rets << ret + +// } + + } + } + return rets; + } + + + def findLogs6(def city){ + def q = new BasicDBObject() + q.put("user_id","1007751") + q.put("parameter_map.city",city) + q.put("request_uri" ,"/property/evaluateForTest") + q.put("c_date" ,new BasicDBObject('$gte', DateUtils.parseFromTightYYYYMMDD("2021-01-19"))) + def rets = [] + def houses = LOG.find(q) + if(houses){ + for(def map : houses){ + if(map.msg=="暂时不支持该小区的估价"){ + continue + } + + def resMap = map.res_map.result + println resMap + if(resMap && map.code == 200 && resMap.calcType==2){ + def ret = [] + //城市 区域 楼盘名称 总价 单价 计算方式 模型特征(经纬度,建筑年代,附近小区,距离,近半年涨跌幅,同区域20%,80%价位) + ret << resMap.city + ret << resMap.district + ret << resMap.name + ret << resMap.totalprice + ret << resMap.unitPrice + ret << resMap.calcType + + def paramMap = map.parameter_map + ret << paramMap.calDate + + rets << ret + } + } + } + return rets; + } + + + public static String handleIllegalCharacter(String str) { + str = str.replaceAll("[\\x00-\\x08\\x0b-\\x0c\\x0e-\\x1f]", ""); + str = str.replace("?","#") + str = str.trim(); + return str; } def findPoiLogs(def city){ @@ -165,20 +739,20 @@ class EsHouseStat { ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) def rets = [] while (rs.next()) { - def date = rs.getDate("m_date") - def type = rs.getString("typecode") - if(["141202", "141203"].contains(type)){ - def ret = [] - ret << rs.getString("cityname") - ret << rs.getString("adname") - ret << rs.getString("name") - ret << rs.getString("address") - ret << rs.getString("type") - ret << rs.getString("typecode") - ret << rs.getFloat("location.lon") - ret << rs.getFloat("location.lat") - rets << ret - } + def date = rs.getDate("m_date") + def type = rs.getString("typecode") + if(["141202", "141203"].contains(type)){ + def ret = [] + ret << rs.getString("cityname") + ret << rs.getString("adname") + ret << rs.getString("name") + ret << rs.getString("address") + ret << rs.getString("type") + ret << rs.getString("typecode") + ret << rs.getFloat("location.lon") + ret << rs.getFloat("location.lat") + rets << ret + } } return rets; } @@ -200,7 +774,7 @@ class EsHouseStat { if(hits.hits){ def arr = hits.hits if(arr){ - def source = arr[0]._source + def source = arr[0]._source if(source && source.code == 200){ def ret = [] def parameter_map = source.parameter_map @@ -284,8 +858,19 @@ class EsHouseStat { def exportOpenLog(){ - def rets = findLogs3(); - ExcelUtils.write("D:\\数据\\东莞银行总行.xlsx", rets,"数据") + def city = "苏州" + def rets = findLogs4(city); + ExcelUtils.write("D:\\数据\\深圳建行正式("+city+").xlsx", rets,"成功") + } + def exportMongoLog(){ + //"上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州", + def citys = ["深圳"] + for(def city :citys){ + def rets = findLogs5(city); +// def rets = findLogs6(city); + ExcelUtils.write("D:\\数据\\深圳建行\\深圳建行正式("+city+").xlsx", rets,"成功") + } + } def exportPoi(){ @@ -309,4 +894,276 @@ class EsHouseStat { } } + + def cityStat(){ + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("request_uri", "/property/evaluateForTest") + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def map = [:] + for(def d : cursor){ + def resMap = d.parameter_map + if(resMap){ + def count = map.get(resMap.city) + if(!count){ + count = 0 + } + map.put(resMap.city, count+1) + + }else{ + println d + } + } + + println map.toString() + } + + + def stat(){ +// def citys = ["深圳"] + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def rets = [] + rets << ["城市","估价总调用量","成功调用量","失败调用量","有样本调用量","无样本调用量"] + for(def city :citys){ + //def datas = ExcelUtils.read("D:\\数据\\深圳建行\\深圳建行正式("+city+").xlsx",0,1) + + def map = findHousePriceType(city, 1); + def ret = [] + ret << city + ret << findTotalCount(city); + ret << findLogs6(city, ["ok"]); + ret << findLogs6(city, ["该小区不存在","暂时不支持该小区的估价"]); + ret << findHouseType(city, 1); + ret << findHouseType(city, 2); + + + ret << map.totalPrice + ret << map.avgPrice + println ret + rets << ret + } + + ExcelUtils.write("D:\\数据\\深圳建行\\深圳建行调用统计2.xlsx", rets, "统计") + } + def findSuccess(def datas){ + def count = 0 + for(def data : datas){ + def msg = data['N'] + if(!msg){ + count++ + } + } + return count + } + + def findFail(def datas){ + def count = 0 + for(def data : datas){ + def msg = data['N'] + if(msg=="该小区不存在" || msg =="暂时不支持该小区的估价"){ + count++ + } + } + return count + } + + def findHouse(def datas, def value){ + def count = 0 + for(def data : datas){ + def type = data['T'] + if(type == value){ + count++ + } + } + return count + } + + def findTotalPrice(def datas, def value){ + def count = 0 + def totalPrice = 0 + for(def data : datas){ + def type = data['T'] + def price = data['R'] + if(type == value){ + count++ + totalPrice = totalPrice + price + } + } + + def map = [:] + map.totalPrice = totalPrice + if(count){ + map.avgPrice = (totalPrice/count).toInteger() + } + return map + } + + def matchStat(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def datas = [] + + datas << ["城市","区域","名称"] + for(def city : citys){ + def rets = findLogs5(city); + def keys = [] + def priceMap = [:] + def cityDatas = [] + for(def ret : rets){ + if(ret.size()>=14 && ret[14] && ret[15] && ret[16]){ + def key = ret[14]+"#"+ret[15]+"#"+ret[16] +// def prices = priceMap.get(key) +// if(!prices){ +// prices = [] +// priceMap.put(key, prices) +// } +// prices << ret[18] +// if(!keys.contains(key)){ +// keys << key + cityDatas << [ret[14], ret[15], ret[16], ret[12],ret[18]] +// } + } +// if(ret.size()==14 && ret[13]=="暂时不支持该小区的估价"){ +// def key = ret[0]+"#"+ret[1]+"#"+ret[2] +// if(!keys.contains(key)){ +// keys << key +// datas<<[ret[0], ret[1], ret[2], ret[12]] +// } +// } + + } + +// for(def k : keys){ +// def arrs = k.split("#") +// datas<<[arrs[0], arrs[1], arrs[2]] +// } + + +// for(def o : cityDatas){ +// def prices = priceMap.get(o[0]+"#"+o[1]+"#"+o[2]); +// def avg_price = prices.sum()/prices.size() +// o << avg_price.toInteger() +// } +// + datas.addAll(cityDatas) + +// def data = [] +// data << city +// data << keys.size() +// datas << data + } + ExcelUtils.write("D:\\数据\\深圳建行\\建行匹配小区(2).xlsx",datas,"统计") + } + + def statNoHouseCalType(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def results = [] + results << ["城市","无样本总量","1","2","3","4","5","6","7","8","9","10","其它"] + for(def city : citys){ + def rets = findLogs6(city); + println rets.size() + + def map = [:] + for(def r : rets){ + def weekend = DateUtils.getFirstDayOfWeek(DateUtils.parseFromTightYYYYMMDD(r[6])) + def obj = CommonDatautils.queryDistrictPrice(r[0], r[1], r[2], weekend) + if(obj){ + def type = map.get(obj.value_type) + if(!type){ + type = 0 + } + map.put(obj.value_type, type+1); + } + } + println city +" "+rets.size() + map.size() + + def rows = [] + rows << city + rows << rets.size() + for(def i=1;i<=10;i++){ + if(map.containsKey(i)){ + rows << map.get(i) + }else{ + rows << "" + } + } + results << rows + } + ExcelUtils.write("D:\\数据\\深圳建行\\无样本计算方式统计.xlsx",results, "数据") + } + + def statCityHouse(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + def rets = [] + rets << ["城市","总量","anjuke.com","58.com","ke.com","leju.com","lianjia.com","c21.com","fang.com","centanet.com","fangdd.com","qfang.com","fccs.com","house365.com","zhuge.com","izhiliao.com","leyoujia.com","5i5j.com","均价"] + def roots = ["anjuke.com","58.com","ke.com","leju.com","lianjia.com","c21.com","fang.com","centanet.com","fangdd.com","qfang.com","fccs.com","house365.com","zhuge.com","izhiliao.com","leyoujia.com","5i5j.com"] + for(def city : citys){ + def ret = [] + ret << city + def total = findHouseBySource(city, null); + ret << total + for(def root : roots){ + def count = findHouseBySource(city, root); + + def pre = DataHandleUtils.getDoubleRound(2, count.toInteger()*100/total.toInteger()) + ret << count+"(${pre}%)" + } + def cityPrice = CommonDatautils.findCityPrice(city, "2021-01-31") + ret << cityPrice + rets << ret + println ret + } + ExcelUtils.write("D:\\数据\\深圳建行\\建行房源统计.xlsx",rets,"统计") + + } + + + + def findHouseBySource(def city, def root){ + final StringBuffer sqlBuffer = new StringBuffer() + sqlBuffer.append("SELECT count(*) as count FROM \"seawater-20*\" WHERE 1=1") + sqlBuffer.append(" and city = '"+city+"'") + sqlBuffer.append(" and cat = 'sell'") + sqlBuffer.append(" and p_date >=20200101") + if(root){ + sqlBuffer.append(" and root = '"+root+"'") + } + ResultSet rs = dataSource.connection.createStatement().executeQuery(sqlBuffer.toString()) + def count = 0 + while (rs.next()){ + count = rs.getString("count") + } + return count; + } + + + def findTestDatas(def city, def value) { + def q = new BasicDBObject() + q.put("user_id", "1007751") + q.put("parameter_map.city", city) + q.put("request_uri", "/property/evaluateForTest") + + def keys = new BasicDBObject() + keys.put("res_map",1) + def cursor = LOG.find(q) + def count = 0 + println "总量:"+cursor.count() + for(def d : cursor){ + def resMap = d.res_map.result + if(resMap){ + if(resMap.calcType==value){ + + } + } + } + return count; + } + + def statNoHouseCCBData(){ + def citys = ["上海","北京","成都","重庆","苏州","广州","大连","厦门","杭州","深圳"] + for(def city : citys){ + findTestDatas(city, 2) + } + } } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy new file mode 100644 index 000000000..7c630dfa9 --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/LogStat.groovy @@ -0,0 +1,117 @@ +package com.fangjia.script.gj + +import com.fangjia.core.util.DateUtil +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils +import com.fangjia.script.util.ElasticUtils +import com.fangjia.script.util.ExcelUtils + +class LogStat { + + static main(args) { + def task = new LogStat() + task.run() + } + + def run(){ + def datas = ExcelUtils.read("D:\\数据\\建行\\建行匹配小区2.xlsx", 0, 1) + println datas.size() + ExcelUtils.multiThreadingModify("D:\\数据\\建行\\建行匹配小区2.xlsx",{ d -> + def calDate = DateUtil.parseDate(d['D'], "yyyy-MM-dd") + def weekend = DateUtils.getFirstDayOfWeek(calDate) + def dbo = CommonDatautils.queryDistrictPrice(d['A'], d['B'], d['C'], weekend) + if(dbo){ + d['E'] = dbo.avg_price + } + + def district = CommonDatautils.findDistrict(d['A'], d['B'], d['C'], "district") + if(district){ + def startDate = DateUtil.addMonth(calDate, -6) + def start = startDate.replace("-","").toInteger() + def end = DateUtil.formatByPattern(calDate,"yyyyMMdd").toInteger() + def alias = [] + if(district.alias){ + if(!district.alias.contains(district.name)){ + district.alias << district.name + } + alias = district.alias + }else{ + alias << district.name + } + def count = findHouses(d['A'], d['B'], d['C'], district.alias, start, end) + d['G'] = count + } + },0) + } + + + def findHouses(def city, def region, def name, def alias, def startDate, def endDate){ + + String sql = "select * from seawater-20* where cat=sell and city='${city}' and region='${region}' and district_name in (${alias.join(',')})" + def rets = [] + ElasticUtils.executeSqlQuery(sql,{data-> + rets << data + println data + }) + return rets.size() + } + +// def findTestDatas(def city, def value) { +// def q = new BasicDBObject() +// q.put("user_id", "1007751") +// q.put("parameter_map.city", city) +// q.put("request_uri", "/property/evaluateForTest") +// +// def keys = new BasicDBObject() +// keys.put("res_map",1) +// def cursor = LOG.find(q) +// def count = 0 +// println "总量:"+cursor.count() +// def datas = [] +// for(def d : cursor){ +// if(d && d.code==200 && d.res_map && d.res_map.code==200){ +// def resMap = d.res_map.result +// if(resMap.calcType==value){ +// def data = [] +// def rowMap = [:] +// rowMap.city = resMap.city +// rowMap.region = resMap.district +// rowMap.name = resMap.name +// rowMap.calDate = d.parameter_map.calDate +// data << rowMap +// } +// } +// } +// return datas; +// } + +// for(def i=0;i3000){ + info1 = null + } + }else{ + info1 = null + } + } + + + if(midSchools) { + info2 = selectSchool(dist.lng / 10000000000.0, dist.lat / 10000000000.0, midSchools); + if(info2 && info2.school_lng2){ + double lat = dist.lat/10000000000.0 + double lng = dist.lng/10000000000.0 + distance2 = GeoHelper.GetDistance(lat, lng, info2.school_lat2, info2.school_lng2) + if(distance2>3000){ + info2 = null + } + }else{ + info2 = null + } + } + if(info1|| info2){ + def row = [] + row << cityCodeMap.get(dist.city) + row << regionCodeMap.get(dist.city +"-"+dist.region) + row << dist.name + row << dist.lng/10000000000.0 + row << dist.lat/10000000000.0 + row << dist.address + if(info1){ + row << info1.school_name + row << info1.school_address!=null?info1.school_address:'' + row << info1.attribute!=null?info1.attribute:'' + row << info1.school_lng2!=null?info1.school_lng2:'' + row << info1.school_lat2!=null?info1.school_lat2:'' +// row << distance1 + }else{ + row << "" + row << "" + row << "" + row << "" + row << "" +// row << "" + } + + if(info2){ + row << info2.school_name + row << info2.school_address!=null?info2.school_address:'' + row << info2.attribute!=null?info2.attribute:'' + row << info2.school_lng2!=null?info2.school_lng2:'' + row << info2.school_lat2!=null?info2.school_lat2:'' +// row << distance2 + }else{ + row << "" + row << "" + row << "" + row << "" + row << "" +// row << "" + } + row << "" + row << DateUtils.formatDateByPattern(dist.c_date, "yyyy-MM-dd HH:mm:ss") + row << DateUtils.formatDateByPattern(dist.m_date, "yyyy-MM-dd HH:mm:ss") + row << "fjw" + row << dist._id.toString() + row << "对口" + + rets << row + } + } + } catch (Exception e) { + e.printStackTrace() + } finally { + latch.countDown() + } + }; + }) + } + latch.await() + ExcelUtils.write("d:\\数据\\楼盘对口学校_V2.xlsx", rets, "数据") + + } + + + def selectSchool(lng, lat, def schools){ + def minSchool = null + def level0 = []//国家 + def level1 = []//省重点 + def level2 = []//市重点 + def level3 = []//区重点 + def level4 = []//普通 + def level5 = []//其它 + for(def s : schools){ + if(s.attribute){ + if(s.attribute.contains("国家重点")){ + level0 << s + }else if(s.attribute.contains("省重点")){ + level1 << s + }else if(s.attribute.contains("市重点")){ + level2 << s + }else if(s.attribute.contains("区重点")){ + level3 << s + }else if(s.attribute.contains("普通")){ + level4 << s + }else{ + level5 << s + } + }else{ + level5 << s + } + } + + + if(level0){ + if(level0.size()==1){ + minSchool = level0.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level0) + } + }else if(level1){ + if(level1.size()==1){ + minSchool = level1.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level1) + } + }else if(level2){ + if(level2.size()==1){ + minSchool = level2.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level2) + } + }else if(level3){ + if(level3.size()==1){ + minSchool = level3.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level3) + } + }else if(level4){ + if(level4.size()==1){ + minSchool = level4.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level4) + } + }else if(level5){ + if(level5.size()==1){ + minSchool = level5.get(0) + }else{ + minSchool = selectMinDistanceSchool(lat, lng, level5) + } + } + + def rets = [] + + if(minSchool){ + rets = findSchoolInfo(minSchool.city, minSchool.match_region,minSchool.school_name) + } + + if(rets.size()>0){ + for(def d : rets){ + if(!minSchool.school_address){ + minSchool.school_address = d.address + } + if(!minSchool.attribute){ + minSchool.attribute = d.attribute + } + } + }else{ + if(minSchool){ + if(minSchool.school_map_lng){ + minSchool.school_lng2 = minSchool.school_map_lng + minSchool.school_lat2 = minSchool.school_map_lat + }else if(minSchool.school_lng1){ + minSchool.school_lng2 = minSchool.school_lng1 + minSchool.school_lat2 = minSchool.school_lat1 + }else if(minSchool.school_lng){ + minSchool.school_lng2 = minSchool.school_lng + minSchool.school_lat2 = minSchool.school_lat + } + } + + } + return minSchool; + } + + + def findSchoolInfo(def city, def region, def schoolName){ + def q = new BasicDBObject() + q.put("city", city) + q.put("region", region) + q.put("name", schoolName) + def dbcursor = school_col.find(q) + def rets = [] + for(def obj : dbcursor){ + rets << obj + } + return rets + } + + def selectMinDistanceSchool(def lat, def lng, def schools){ + def minSchool = null; + def minDistance = Integer.MAX_VALUE + for(def s1 : schools){ + + def lng2 = null + def lat2 = null + if(s1.school_map_lng){ + lng2 = s1.school_map_lng + lat2 = s1.school_map_lat + }else if(s1.school_lng1){ + lng2 = s1.school_lng1 + lat2 = s1.school_lat1 + }else if(s1.school_lng){ + lng2 = s1.school_lng + lat2 = s1.school_lat + } + if(lat2){ + def distance = GeoHelper.GetDistance(lat, lng, lat2, lng2) + println s1.district_name +" "+ s1.school_name+" "+distance + if(distancetotalPage){ + stop = true; + } + schools.addAll(data.list) + } + } + } + } + + print(schools.size()); + + def districtSchools = [] + def schoolInfos = [] + if(schools.size()>0){ + + def url = "https://51yongka.com/fapi/school/details" + for(def d : schools){ + def params = "schoolId="+d.id+"&access_token=oRqu15Mq_KxHfK0y0ZWhwktVnfUM" + def s = chaxuequInfoCrawl(url, params) + JSONObject jsonObject = JSONObject.fromObject(s); + if(jsonObject && jsonObject.code==200){ + + if(jsonObject.data && jsonObject.data.nearbyPropertyList){ + def districts = jsonObject.data.nearbyPropertyList + for(def dist : districts){ + def dataMap = new BasicDBObject() + dataMap.city = jsonObject.data.city + dataMap.region = RegionData.formatRegion(dataMap.city , jsonObject.data.area) + /**类型: 幼儿园,小学,初中*/ + dataMap.category = converCategory(jsonObject.data.schoolName) + dataMap.district_name = dist.name.trim() + dataMap.district_address = dist.address + dataMap.school_name = jsonObject.data.schoolName.trim() + dataMap.school_address = jsonObject.data.address + dataMap.source = "查学区房" + dataMap.url = url + "?"+params + dataMap.telephone = jsonObject.data.officialTelephone; + districtSchools << dataMap + } + }else{ + + def schoolData = new BasicDBObject() + schoolData.province = "江苏" + schoolData.city = jsonObject.data.city + schoolData.region = RegionData.formatRegion(schoolData.city , jsonObject.data.area) + /**类型: 幼儿园,小学,初中*/ + schoolData.category = converCategory(jsonObject.data.schoolName) + schoolData.name = jsonObject.data.schoolName.trim() + schoolData.address = jsonObject.data.address + /**来源网站*/ + schoolData.source = "查学区房" + /**房源详情url*/ + schoolData.telephone = jsonObject.data.officialTelephone; + schoolInfos << schoolData + println jsonObject.data.schoolName +" 无周边楼盘" + } + + } + println s; + } + } + + println "小区:"+districtSchools.size() +// saveChaxuequDistrictSchool(districtSchools) + saveChaxuequSchool(schoolInfos) + + } + + def saveChaxuequSchool(schoolInfos){ + for(def ds : schoolInfos){ + def obj = findSchoolData(ds.city, ds.school_name); + if(!obj){ + school_col.save(ds) + } + } + } + + + + + def saveChaxuequDistrictSchool(def districtSchools){ + for(def ds : districtSchools){ + def obj = findDistrictSchoolData(ds.city, ds.school_name); + if(!obj){ + district_school_col.save(ds) + } + } + } + + def findSchoolData(city, school_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("school_name", school_name) + return school_col.findOne(q) + } + + + def findDistrictSchoolData(city, school_name, district_name){ + def q = new BasicDBObject() + q.put("city", city) + q.put("school_name", school_name) + q.put("district_name", district_name) + return district_school_col.findOne(q) + } + + def converCategory(def schoolName){ + + if(schoolName){ + if(schoolName.contains("初中部")||schoolName.contains("中学")){ + return "初中" + }else if(schoolName.contains("小学")){ + return "小学" + } + } + return null + } + + def static chaxuequInfoCrawl(url, bodyquery){ + def responseMap = [:] + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","51yongka.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","75") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("content-type","application/x-www-form-urlencoded") + headers.put("Referer","https://servicewechat.com/wxc2461b2529576950/8/page-frame.html") + headers.put("Accept-Encoding","gzip, deflate, br") + String s = doPost(url, bodyquery, headers) + return s + } catch (Exception e) { + e.printStackTrace() + } + return null + } + + + def static schoolList(page) { + def responseMap = [:] + String s = null + try { + //headers信息 + Map headers = new LinkedHashMap() + headers.put("Host","m.gusuwang.com") + headers.put("Connection","keep-alive") + headers.put("Content-Length","40") + headers.put("User-Agent","Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.143 Safari/537.36 MicroMessenger/7.0.9.501 NetType/WIFI MiniProgramEnv/Windows WindowsWechat") + headers.put("content-type","application/x-www-form-urlencoded") + headers.put("Referer","https://servicewechat.com/wx9188a941e5ef95c5/10/page-frame.html") +// headers.put("Accept-Encoding","gzip, deflate, br") + + //params信息 + Map params = new HashMap(); + params.put("secCode","198a53da6baa3aa6a5b99959a006be37"); +// //post请求 + def url = "https://m.gusuwang.com/xuequ/xqapi.php?act=searchschool" + s = doPost(url, "secCode=198a53da6baa3aa6a5b99959a006be37&keys=&grade=&lb=&lx=&area=&currpage="+page,headers) + } catch (Exception e) { + e.printStackTrace() + } + return s + } + + + + + + + static String doPost(String strUrl, String content, Map headers) { + String result = ""; + OutputStream outputStream = null; + InputStream inputStream = null; + StringBuilder sb = new StringBuilder(); + try { + URL url = new URL(strUrl); + //通过调用url.openConnection()来获得一个新的URLConnection对象,并且将其结果强制转换为HttpURLConnection. + HttpURLConnection urlConnection = (HttpURLConnection) url.openConnection(); + urlConnection.setRequestMethod("POST"); + //设置连接的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setConnectTimeout(30000); + //设置读取的超时值为30000毫秒,超时将抛出SocketTimeoutException异常 + urlConnection.setReadTimeout(30000); + //将url连接用于输出,这样才能使用getOutputStream()。getOutputStream()返回的输出流用于传输数据 + urlConnection.setDoOutput(true); + urlConnection.setDoInput(true); + //设置通用请求属性为默认浏览器编码类型 +// urlConnection.setRequestProperty("content-type", "application/x-www-form-urlencoded"); + + + if (headers != null && headers.keySet().size() > 0) { + for (String head : headers.keySet()) { + urlConnection.setRequestProperty(head, headers.get(head)); + } + } + //getOutputStream()返回的输出流,用于写入参数数据。 + outputStream = urlConnection.getOutputStream(); + outputStream.write(content.getBytes()); + outputStream.flush(); + outputStream.close(); + //此时将调用接口方法。getInputStream()返回的输入流可以读取返回的数据。 + inputStream = urlConnection.getInputStream(); + + +// byte[] data = new byte[1024]; +// //inputStream每次就会将读取1024个byte到data中,当inputSteam中没有数据时,inputStream.read(data)值为-1 +// while (inputStream.read(data) != -1) { +// String s = new String(data, Charset.forName("utf-8")); +// sb.append(s); +// } + + InputStreamReader reader = new InputStreamReader(inputStream, "UTF-8"); + char[] buff = new char[1024]; + int length = 0; + while ((length = reader.read(buff)) != -1) { + String s = new String(buff, 0, length); + sb.append(s); + } + } catch (IOException e) { + e.printStackTrace(); + }finally{ + if(outputStream!=null){ + outputStream.close(); + } + if(inputStream!=null){ + inputStream.close(); + } + } + + return sb.toString(); + } + + + + + + +} diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy index 62c388a22..e80e49ac4 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticUtils.groovy @@ -52,7 +52,7 @@ class ElasticUtils { return dataSource; } - static executeSqlQuery(String sql, Closure closure, String host = "192.168.0.216"){ + static executeSqlQuery(String sql, Closure closure, String host = "192.168.0.206"){ EsDataSource dataSource = getEsDataSource(host) Connection connection = dataSource.getConnection() Statement statement = connection.createStatement()