升级Es驱动版本

This commit is contained in:
2021-04-08 13:32:52 +08:00
parent e4a18340bb
commit 32d1f77ec7
54 changed files with 361 additions and 453 deletions
+7 -22
View File
@@ -17,8 +17,8 @@
<artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
<source>1.8</source>
<target>1.8</target>
<encoding>GBK</encoding>
</configuration>
</plugin>
@@ -179,7 +179,7 @@
<dependency>
<groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId>
<version>4.4.1</version>
<version>4.5.10</version>
</dependency>
<dependency>
<groupId>org.apache.ant</groupId>
@@ -315,16 +315,6 @@
<artifactId>metadata-extractor</artifactId>
<version>2.4.0</version>
</dependency>
<dependency>
<groupId>org.apache.http</groupId>
<artifactId>httpasyncclient</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>org.apache.http</groupId>
<artifactId>httpcore-nio</artifactId>
<version>4.1.1</version>
</dependency>
<dependency>
<groupId>org.apache.sanselan</groupId>
<artifactId>sanselan</artifactId>
@@ -448,23 +438,18 @@
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>x-pack-transport</artifactId>
<version>6.4.2</version>
<artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>7.11.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.plugin</groupId>
<artifactId>x-pack-sql-jdbc</artifactId>
<version>6.4.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>6.4.2</version>
<version>7.11.2</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId>
<version>7.4.0</version>
<version>8.7.0</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
@@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
def findHouseMaxArea(String city, String region, String name, List alias) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24)
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection
import com.mongodb.client.model.UpdateOptions
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it->
names << it.name
}
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
def findHouseCount(String city, String region, String name, List alias, int months) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months)
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement;
import java.sql.Statement
/**
*
* 历史均价计算
@@ -27,7 +23,7 @@ import java.sql.Statement;
*/
@BatchOwner(Watcher.JiangYaHui)
class HistoricalAverageRevaluationTask extends BaseBatch{
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat')
@@ -1,6 +1,6 @@
package com.fangjia.script.avgprice
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
@@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement;
import com.mongodb.*
/**
* 无样本小区历史遗漏数据补充
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
@@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -4,13 +4,13 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class AddEsHouseField {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
@@ -63,7 +62,7 @@ class AddThirdPriceDistrict {
def province = null
def resAddrMap = [:]
def filterEstateTypeMap = [:]
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,11 +1,11 @@
package com.fangjia.script.gj
import com.esms.common.util.CommonUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import com.huaban.analysis.jieba.JiebaSegmenter
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -485,7 +485,7 @@ class CrawlDataMatch {
}
def findAllAddress(def cityName){
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,24 +1,19 @@
package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.SQLException
import java.sql.Statement
class EsHouseStat {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,12 +1,12 @@
package com.fangjia.script.gj
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class EsSearchUtils {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -20,7 +19,7 @@ class FZDataExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -3,14 +3,14 @@ package com.fangjia.script.gj
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class RoomStat {
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,30 +1,25 @@
package com.fangjia.script.gj
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.zip.GZIPInputStream
import static org.elasticsearch.index.query.QueryBuilders.idsQuery
class TradeDataMatch {
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0)
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
File diff suppressed because one or more lines are too long
@@ -1,24 +1,17 @@
package com.fangjia.script.gj.ccb
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet
import java.text.SimpleDateFormat
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -26,12 +19,11 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.script.util.FangJiaAreaMapUtils
class HouseExport {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
class ExportData {
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.*
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -27,7 +22,7 @@ class DistrictWash {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
static def totalCount = new AtomicLong()
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils;
import com.fangjia.script.util.DateUtils;
import com.fangjia.script.util.ExcelParser
import com.fangjia.script.common.DataHandleUtils;
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.*
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.io.FileUtils;
import org.bson.types.ObjectId
import com.fangjia.script.util.MongoUtils;
import com.mongodb.QueryOperators
import com.fangjia.script.util.OpenApiClient
import java.util.Map.Entry;
import net.sf.json.JSONObject
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import org.apache.commons.lang.StringUtils
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.Map.Entry
import java.util.regex.Matcher
import java.util.regex.Pattern
class CZBlankDataExport {
@@ -62,7 +52,7 @@ class CZBlankDataExport {
def directionMap = [1:"ÄÏ",2:"±±",3:"Äϱ±",4:"¶«",5:"Î÷",6:"¶«Î÷",7:"¶«ÄÏ",8:"Î÷ÄÏ",9:"¶«±±",10:"Î÷±±"]
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
/**
* 江南银行线上数据交付
@@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.CrawlCommandConsumerBatch
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.BatchUpdateException
import java.sql.ResultSet
import java.text.NumberFormat
import java.util.Map.Entry
import java.util.concurrent.BlockingQueue
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.LinkedBlockingDeque
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit
import java.util.regex.Pattern
@BatchOwner(Watcher.WeiLei)
class HousingOverviewExportBatch extends BaseBatch {
static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>())
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,6 +1,5 @@
package com.fangjia.script.jusfoun
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.DateUtils
@@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import redis.clients.jedis.JedisPool
import redis.clients.jedis.JedisPoolConfig
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.*
import java.text.DateFormat
@@ -56,7 +53,7 @@ class JusfounCommon {
public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
private static JdbcDataSource dataSource = null
private static EsDataSource dataSource = null
private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>();
private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>();
@@ -390,7 +387,7 @@ class JusfounCommon {
return DataUtil.hanyuQuanpin(hanzi, '')
}
static JdbcDataSource getESDataSource() {
static EsDataSource getESDataSource() {
if (dataSource != null) {
return dataSource;
} else {
@@ -399,7 +396,7 @@ class JusfounCommon {
return dataSource;
}
try {
dataSource = new JdbcDataSource();
dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties();
@@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import static com.mongodb.client.model.Filters.*
def List find_rent_house_prices(String city, String region, String name, Date start, Date end){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,17 @@
package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui)
class MongoDBLogDisMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -1,20 +1,18 @@
package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui)
class MongoDBLogMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.atomic.AtomicInteger
@@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch {
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource()
private static final EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -53,8 +53,10 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
void process(Object args) {
Map userInfo = getUserInfo()
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
def endDate = DateUtil.truncateDay(new Date())
// def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
// def endDate = DateUtil.truncateDay(new Date())
def startDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210308"))
def endDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210314"))
BasicDBObject query = new BasicDBObject()
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
@@ -64,7 +66,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
long count = 0
long total = cursor.count()
print("total="+total)
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia")
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("seawater.es.db.fangjia.com")
for(BasicDBObject log : cursor){
Map<String,Object> parameter_map = (Map) log.get("parameter_map")
if(parameter_map == null) {
@@ -128,7 +130,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
cmd.put("user_name", user.user_name)
cmd.put("company_name", user.company)
}
bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd))
bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd))
println(cmd)
count++
if(count % 100 == 0){
@@ -1,50 +1,24 @@
package com.fangjia.script.trade
///µ¼Èë¼Û¸ñÖ¸Êý
import com.fangjia.script.BaseBatch;
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
///导入价格指数
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.xssf.usermodel.XSSFCell
import org.apache.poi.xssf.usermodel.XSSFWorkbook
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.QueryOperators
import com.fangjia.script.util.DateUtils;
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.Mongo
import groovy.json.JsonSlurper
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.RandomUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.NumberFormat;
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.core.util.DBIterator
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.StringUtil;
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils;
import com.ibm.icu.text.SimpleDateFormat;
import com.fangjia.storehouse.DBIterator
import com.fangjia.storehouse.DBIterator;
import com.fangjia.storehouse.StorehouseDB;
import org.apache.commons.lang.math.RandomUtils;
@BatchOwner(Watcher.GaoJuan)
class ImportIndexStats extends BaseBatch{
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
@@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{
public static final def dingjiYear = 2016;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource()
private static final EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,17 +1,12 @@
package com.fangjia.script.trade
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -22,7 +17,7 @@ class TradeDealPriceStat {
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -3,7 +3,7 @@ package com.fangjia.script.usahouse
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
class AddressMatch {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,6 +1,6 @@
package com.fangjia.script.usahouse
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
@@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSON
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.select.Elements
@@ -1,69 +1,125 @@
package com.fangjia.script.util
import org.apache.commons.lang3.SystemUtils
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.TransportAddress
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticsearchHelper {
class ElasticsearchHelper {
static getClient(String host, String clusterName = "fangjia"){
Settings.Builder builder = Settings.builder()
.put("cluster.name", clusterName)
.put("client.transport.sniff", false)
.put("client.transport.ping_timeout", "60s")
.put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3")
.put("xpack.security.transport.ssl.enabled", "true")
if(SystemUtils.IS_OS_WINDOWS){
builder
.put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
.put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
}else{
builder
.put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12")
.put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12")
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
Properties connectionProperties = new Properties();
connectionProperties.setProperty("user","developer");
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build()
Settings settings = builder.build()
return new PreBuiltXPackTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
rs.close()
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 60);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param clusterName
* @param host
* @return
*/
static createBulkProcessor( String host, String clusterName){
TransportClient client = getClient(host, clusterName)
BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {
static createBulkProcessor(String host) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request, Throwable failure) {
failure.printStackTrace()
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
}).setBulkActions(10000).setConcurrentRequests(20).build()
};
RestHighLevelClient client = getClient(host);
return bulkProcessor
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
/**
@@ -71,14 +127,13 @@ class ElasticsearchHelper {
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor){
static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.MINUTES)
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
}
@@ -1,6 +1,6 @@
package com.fangjia.script.weilei
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.Bytes
import com.mongodb.DBCollection
@@ -8,7 +8,7 @@ import com.mongodb.DBCursor
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -16,7 +16,7 @@ import java.sql.SQLException
import java.sql.Statement
import java.text.SimpleDateFormat
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties();
@@ -1,31 +1,16 @@
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.CommandResult
import com.mongodb.DB
import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*;
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
@@ -43,7 +28,7 @@ cityList.each { def city ->
}
})
roots.each { def root ->
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,23 +1,12 @@
package com.fangjia.script.wenyan.address
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.util.QQKeyPool
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -44,7 +33,7 @@ import java.sql.Statement
// data << (k.split("_") as List)
//}
//ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û")
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,20 @@
package com.fangjia.script.wenyan.avgPrice
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
def findEstateType(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math3.ml.clustering.CentroidCluster
import org.apache.commons.math3.ml.clustering.DoublePoint
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017)
def findHouse(BasicDBObject district){
@@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){
sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ")
sqlBuffer.append(" ORDER BY p_date desc")
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,12 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
//
//def data = [["城市", "区域", "小区名", "2018", "全部"]]
//cityList = ["上海"]
@@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq
//HttpRequest httpRequest = new HttpRequest()
//println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8")
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config")
AVERAGER_CONFIG.find().each {Document doc->
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20201101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20201201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -1,19 +1,13 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -61,7 +55,7 @@ import java.sql.Statement
//println(bigMarkupCount)
def countHouses(String city, String region, String name){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,28 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.bson.Document
import com.mongodb.client.MongoCollection
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def poiData = Collections.synchronizedList([])
//
//ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item ->
@@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([])
// println(item)
//},1)
def findHouseCount(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -1,11 +1,17 @@
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
//import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
@@ -132,4 +138,70 @@ import org.apache.commons.lang3.StringUtils
// item["E"] = RegionData.formatRegion(item["B"], item["C"])
// println(item)
//})
CommonUtils.reFitAverage("常州","新北","龙城花园", "普通住宅")
//def border = [13511506.7,3639168.61,13511516.8,3639404.86,13511515.27,3639416.92,13511738.21,3639384.94,13512007.72,3639336.35,13511984.92,3639074.96,13511722.75,3639121.08,13511506.7,3639168.61]
//def coords = []
//for(int i=0; i < border.size() - 1; i=i+2){
// coords << "${border[i]},${border[i+1]}"
//}
//def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
//println(json)
//def points = []
//for(JSONObject it : json.result){
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// points << "${geo.lng},${geo.lat}"
// println(" new AMap.LngLat(${geo.lng},${geo.lat}),")
//}
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
def coords = []
def baidu_points = []
def gaode_points = []
for(int i=0; i < border.size() - 1; i=i+2){
coords << "${border[i]},${border[i+1]}"
if(coords.size() > 50){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
coords.clear()
}
}
if(coords.size() > 0){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
}
dbo.format_city = RegionData.formatCity(dbo.city)
dbo.baidu_border = baidu_points.join(";")
dbo.gaode_border = gaode_points.join(";")
BAIDUMAP_BORDER_COLL.save(dbo)
println(dbo)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
// if(seaweed){
// item["A"] = seaweed._id.toString()
// def names = [item['D']]
// if(seaweed.alias){
// names.addAll(seaweed.alias)
// }
// names.unique()
// if(seaweed.lng){
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
// }
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
// if(one){
// item["I"] = one.baidu_border
// }
// println(item)
// }
//})
@@ -669,7 +669,6 @@ def findRentInfoByRoom(String city, String region){
def findRent(def city, def region, def name){
DBCollection DISTRICT_STAT_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
def query = new BasicDBObject()
query.city = city
query.region = region
@@ -1,19 +1,8 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\下载记录20200818-结果 - 副本.xlsx",{List item->
@@ -1,23 +1,9 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
//
//ExcelUtils.multiThreadingModify("D:\\数据\\建行学校数据\\上海学校_0811.xls",{List item->
@@ -5,13 +5,12 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
//def data = []
//data << ["城市", "区域", "小区名", "近一月", "近三月", "近6月", "近一年", "户数"]
//DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("上海")).each {BasicDBObject district->
@@ -36,7 +35,7 @@ import java.sql.Statement
//ExcelUtils.write("D:\\上海成交统计.xlsx", data, "成交数据")
def find_rent_house_prices(String city, String region, String name){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -80,7 +79,7 @@ def find_rent_house_prices(String city, String region, String name){
}
ExcelUtils.multiThreadingModify("D:\\上海成交统计.xlsx", {List item->
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,13 +1,12 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.script.util.*
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -16,7 +15,7 @@ import java.sql.Statement
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
def String hasElevator(BasicDBObject seaweed){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,40 +1,7 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\新增小区.xlsx", 0, 0).each {List item->
def keys = new BasicDBObject()
keys.city = 1
keys.region = 1
keys.name = 1
keys.estate_type2 = 1
def query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
if(!DbUtils.findDistrictStat(dbo, new Date())){
CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
println(dbo)
}
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
// println(dbo)
}
}
//["佛山","济南","北京","广州","深圳"].each {def city->
// def keys = new BasicDBObject()
// keys.city = 1
// keys.region = 1
// keys.name = 1
// keys.estate_type2 = 1
// def query = DbUtils.buildSeaweedQuery(city)
// query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
// DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
// println(dbo)
// }
//}
ElasticsearchHelper.executeQuery("select * from seawater where p_date > 20210407 limit 10",{Map row->
println(row.name)
}, "192.168.0.216")
@@ -1,20 +1,14 @@
package com.fangjia.script.yinjihaun.price
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")