This commit is contained in:
2021-04-14 09:48:09 +08:00
133 changed files with 19669 additions and 955 deletions
@@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
def findHouseMaxArea(String city, String region, String name, List alias) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24)
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection
import com.mongodb.client.model.UpdateOptions
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it->
names << it.name
}
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
def findHouseCount(String city, String region, String name, List alias, int months) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months)
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement;
import java.sql.Statement
/**
*
* 历史均价计算
@@ -27,7 +23,7 @@ import java.sql.Statement;
*/
@BatchOwner(Watcher.JiangYaHui)
class HistoricalAverageRevaluationTask extends BaseBatch{
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat')
@@ -1,6 +1,6 @@
package com.fangjia.script.avgprice
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
@@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement;
import com.mongodb.*
/**
* 无样本小区历史遗漏数据补充
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap
@@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -4,13 +4,13 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class AddEsHouseField {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
@@ -63,7 +62,7 @@ class AddThirdPriceDistrict {
def province = null
def resAddrMap = [:]
def filterEstateTypeMap = [:]
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,11 +1,11 @@
package com.fangjia.script.gj
import com.esms.common.util.CommonUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import com.huaban.analysis.jieba.JiebaSegmenter
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -485,7 +485,7 @@ class CrawlDataMatch {
}
def findAllAddress(def cityName){
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,24 +1,19 @@
package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.SQLException
import java.sql.Statement
class EsHouseStat {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,12 +1,12 @@
package com.fangjia.script.gj
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class EsSearchUtils {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -20,7 +19,7 @@ class FZDataExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -3,14 +3,14 @@ package com.fangjia.script.gj
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
class RoomStat {
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,30 +1,25 @@
package com.fangjia.script.gj
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger
import java.util.zip.GZIPInputStream
import static org.elasticsearch.index.query.QueryBuilders.idsQuery
class TradeDataMatch {
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0)
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
File diff suppressed because one or more lines are too long
@@ -1,24 +1,17 @@
package com.fangjia.script.gj.ccb
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet
import java.text.SimpleDateFormat
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService
@@ -26,12 +19,11 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.script.util.FangJiaAreaMapUtils
class HouseExport {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -83,11 +75,11 @@ class HouseExport {
// exportRentCSV(city, districts, 20200201, 20210301, 2020);
}else{
exportRentCSV(city, districts, 20170801, 20181001, 2017)
exportRentCSV(city, districts, 20200201, 20210301, 2020);
// exportRentCSV(city, districts, 20170801, 20181001, 2017)
// exportRentCSV(city, districts, 20200201, 20210301, 2020);
exportSellCSV(city, districts, 20170801, 20181001, 2017);
exportSellCSV(city, districts, 20200201, 20210301, 2020);
// exportSellCSV(city, districts, 20200201, 20210301, 2020);
}
@@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
class ExportData {
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.util.*
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -27,7 +22,7 @@ class DistrictWash {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
static def totalCount = new AtomicLong()
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ExcelUtils
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils;
import com.fangjia.script.util.DateUtils;
import com.fangjia.script.util.ExcelParser
import com.fangjia.script.common.DataHandleUtils;
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.*
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.io.FileUtils;
import org.bson.types.ObjectId
import com.fangjia.script.util.MongoUtils;
import com.mongodb.QueryOperators
import com.fangjia.script.util.OpenApiClient
import java.util.Map.Entry;
import net.sf.json.JSONObject
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.io.FileUtils
import org.apache.commons.lang.StringUtils
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.Map.Entry
import java.util.regex.Matcher
import java.util.regex.Pattern
class CZBlankDataExport {
@@ -62,7 +52,7 @@ class CZBlankDataExport {
def directionMap = [1:"南",2:"北",3:"南北",4:"东",5:"西",6:"东西",7:"东南",8:"西南",9:"东北",10:"西北"]
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
/**
* 江南银行线上数据交付
@@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.CrawlCommandConsumerBatch
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.BatchUpdateException
import java.sql.ResultSet
import java.text.NumberFormat
import java.util.Map.Entry
import java.util.concurrent.BlockingQueue
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.LinkedBlockingDeque
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit
import java.util.regex.Pattern
@BatchOwner(Watcher.WeiLei)
class HousingOverviewExportBatch extends BaseBatch {
static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>())
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,11 +1,10 @@
package com.fangjia.script.jusfoun
import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.ElasticUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import groovy.sql.GroovyRowResult
@@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import redis.clients.jedis.JedisPool
import redis.clients.jedis.JedisPoolConfig
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.*
import java.text.DateFormat
@@ -56,7 +53,7 @@ class JusfounCommon {
public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
private static JdbcDataSource dataSource = null
private static EsDataSource dataSource = null
private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>();
private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>();
@@ -390,7 +387,7 @@ class JusfounCommon {
return DataUtil.hanyuQuanpin(hanzi, '')
}
static JdbcDataSource getESDataSource() {
static EsDataSource getESDataSource() {
if (dataSource != null) {
return dataSource;
} else {
@@ -399,7 +396,7 @@ class JusfounCommon {
return dataSource;
}
try {
dataSource = new JdbcDataSource();
dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties();
@@ -625,13 +622,13 @@ class JusfounCommon {
String scrollID = null) {
try {
if (transportClient == null) {
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
transportClient = ElasticUtils.getClient("192.168.0.206");
}
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
} catch (Exception e) {
e.printStackTrace()
try {
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
transportClient = ElasticUtils.getClient("192.168.0.206");
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
} catch (Exception e1) {
e1.printStackTrace()
@@ -8,21 +8,13 @@ import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import java.sql.BatchUpdateException
import java.util.concurrent.LinkedBlockingDeque
import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit
@BatchOwner(Watcher.WangWenYan)
class RentHouseExportBatch extends BaseBatch {
@@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import static com.mongodb.client.model.Filters.*
def List find_rent_house_prices(String city, String region, String name, Date start, Date end){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,17 @@
package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui)
class MongoDBLogDisMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -1,20 +1,18 @@
package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui)
class MongoDBLogMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.atomic.AtomicInteger
@@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch {
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource()
private static final EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -5,7 +5,7 @@ import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
@@ -55,7 +55,6 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
Map userInfo = getUserInfo()
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
def endDate = DateUtil.truncateDay(new Date())
BasicDBObject query = new BasicDBObject()
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
@@ -64,7 +63,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
long count = 0
long total = cursor.count()
print("total="+total)
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia")
BulkProcessor bulkProcessor = ElasticUtils.createBulkProcessor("seawater.es.db.fangjia.com")
for(BasicDBObject log : cursor){
Map<String,Object> parameter_map = (Map) log.get("parameter_map")
if(parameter_map == null) {
@@ -128,7 +127,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
cmd.put("user_name", user.user_name)
cmd.put("company_name", user.company)
}
bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd))
bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd))
println(cmd)
count++
if(count % 100 == 0){
@@ -1,50 +1,24 @@
package com.fangjia.script.trade
///µ¼Èë¼Û¸ñÖ¸Êý
import com.fangjia.script.BaseBatch;
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
///导入价格指数
import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.poi.xssf.usermodel.XSSFCell
import org.apache.poi.xssf.usermodel.XSSFWorkbook
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.QueryOperators
import com.fangjia.script.util.DateUtils;
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection;
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.Mongo
import groovy.json.JsonSlurper
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.RandomUtils
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.NumberFormat;
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.core.util.DBIterator
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.StringUtil;
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils;
import com.ibm.icu.text.SimpleDateFormat;
import com.fangjia.storehouse.DBIterator
import com.fangjia.storehouse.DBIterator;
import com.fangjia.storehouse.StorehouseDB;
import org.apache.commons.lang.math.RandomUtils;
@BatchOwner(Watcher.GaoJuan)
class ImportIndexStats extends BaseBatch{
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
@@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{
public static final def dingjiYear = 2016;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource()
private static final EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,17 +1,12 @@
package com.fangjia.script.trade
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import com.mongodb.*
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
@@ -22,7 +17,7 @@ class TradeDealPriceStat {
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
static {
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -3,7 +3,7 @@ package com.fangjia.script.usahouse
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
class AddressMatch {
public static JdbcDataSource dataSource = new JdbcDataSource()
public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static {
dataSource.setUrl(address)
@@ -1,6 +1,6 @@
package com.fangjia.script.usahouse
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch
@@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher
import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSON
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.select.Elements
@@ -1,122 +1,170 @@
package com.fangjia.script.util
import org.apache.commons.lang3.SystemUtils
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.action.index.IndexRequest
import org.elasticsearch.action.search.ClearScrollRequest
import org.elasticsearch.action.search.SearchRequest
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchType
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.TransportAddress
import org.elasticsearch.common.unit.ByteSizeUnit
import org.elasticsearch.common.unit.ByteSizeValue
import org.elasticsearch.action.search.SearchScrollRequest
import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestClient
import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.sort.FieldSortBuilder
import org.elasticsearch.search.sort.SortOrder
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticUtils {
private static TransportClient client = null;
public static TransportClient getClient(def host){
if(client != null){
return client;
static getEsDataSource(String host){
EsDataSource dataSource = new EsDataSource ();
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
Properties connectionProperties = new Properties();
connectionProperties.setProperty("user","developer");
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
dataSource.setProperties(connectionProperties);
return dataSource;
}
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
EsDataSource dataSource = getEsDataSource(host)
Connection connection = dataSource.getConnection()
Statement statement = connection.createStatement()
ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
}
def certPath = ""
if(SystemUtils.IS_OS_WINDOWS){
certPath= "D:\\goojiaconfig\\seawater\\conf\\elastic-certificates.p12"
}else{
certPath = "/opt/elasticmongo/elastic-certificates.p12"
rs.getMetaData().getColumnCount()
while(rs.next()){
def row = [:]
for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
Settings.Builder builder = Settings.builder()
.put("cluster.name", "fangjia")
.put("client.transport.sniff", true)
builder.put("xpack.security.user", "developer" + ":" + "5e8edd851d2fdfbd7415232c67367cc3")
.put("xpack.security.transport.ssl.enabled", "true")
.put("xpack.security.transport.ssl.truststore.path", certPath)
.put("xpack.security.transport.ssl.keystore.path", certPath)
.put("xpack.security.transport.ssl.verification_mode", "certificate")
Settings settings = builder.build();
client = new PreBuiltXPackTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
return client
rs.close()
connection.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure, String type= "_doc"){
client = getClient(host);
BulkProcessor bulkProcessor = BulkProcessor.builder(
client,
new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId,
BulkRequest request) {
}
@Override
void afterBulk(long executionId,
BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId,
BulkRequest request,
Throwable failure) {
}
})
.setBulkActions(10000)
.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB))
.setFlushInterval(TimeValue.timeValueSeconds(5))
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10))
.build()
SearchResponse scrollResp = null;
int errorCount = 0;
def hits ;
def stop = false;
while(!stop){
try{
scrollResp = client.prepareSearch(indexName)
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
.setScroll(new TimeValue(600000))
.setQuery(qb)
.setSize(1000).get()
hits = scrollResp.getHits().getHits()
stop = true;
}catch(Exception e){
errorCount++;
if(errorCount>11){
stop = true;
}
println "错误:${errorCount} "+e.getMessage()
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 60);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
}
/**
* 创建一个ES批处理器
* @param host
* @return
*/
static createBulkProcessor(String host, RestHighLevelClient client = null) {
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request,
Throwable failure) {
}
};
if(!client){
client = getClient(host);
}
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
}
if(hits){
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor) {
for (int k=0;k<hits.size();k++) {
final SearchHit hit = hits[k];
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush()
bulkProcessor.close()
}
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
RestHighLevelClient client = getClient(host)
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def nThread = Runtime.getRuntime().availableProcessors() * 5
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
final SearchHit hit = searchHits[k];
executorService.submit(new Runnable() {
@Override
void run() {
@@ -124,96 +172,66 @@ class ElasticUtils {
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
// println(hits)
e.printStackTrace()
}
}
})
}
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
while(scrollResp.getHits().getHits().length != 0){
hits = scrollResp.getHits().getHits();
for (int k=0;k<hits.size();k++) {
final SearchHit hit = hits[k];
executorService.submit(new Runnable() {
@Override
void run() {
try{
if(closure.getMaximumNumberOfParameters() == 1){
def update = closure.call(hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
}
}else if(closure.getMaximumNumberOfParameters() == 2){
def update = closure.call(hit.getId(), hit.getSourceAsMap())
if(update){
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
}
}
}catch (Exception e){
// println(hits)
e.printStackTrace()
}
}
})
}
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
}
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
executorService.shutdown()
executorService.awaitTermination(1, TimeUnit.DAYS)
bulkProcessor.close()
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
client.close()
}
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
client = getClient(host);
SearchResponse scrollResp = null;
int errorCount = 0;
def hits ;
def stop = false;
while(!stop){
try{
scrollResp = client.prepareSearch(indexName)
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
.setScroll(new TimeValue(600000))
.setQuery(qb)
.setSize(10000)
.setScroll(TimeValue.timeValueMinutes(1))//设置 search context 维护1分钟的有效期
.get()
stop = true;
}catch(Exception e){
errorCount++;
if(errorCount>11){
stop = true;
}
println "错误:${errorCount} "+e.getMessage()
}
}
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
RestHighLevelClient client = getClient(host)
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(qb);
searchRequest.source(searchSourceBuilder);
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
String scrollId = searchResponse.getScrollId();
SearchHit[] searchHits = searchResponse.getHits().getHits();
def results = []
while(scrollResp.getHits() && scrollResp.getHits().getHits().length != 0){
for (SearchHit searchHit : scrollResp.getHits().getHits()) {
while (searchHits != null && searchHits.length > 0) {
for (int k=0; k < searchHits.size(); k++) {
SearchHit searchHit = searchHits[k]
searchHit.getSourceAsMap().put("id", searchHit.getId());
results << searchHit.getSourceAsMap()
}
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet();
//当searchHits的数组为空的时候结束循环,至此数据全部读取完毕
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
scrollId = searchResponse.getScrollId();
searchHits = searchResponse.getHits().getHits();
}
//删除scroll
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollResp.getScrollId());
client.clearScroll(clearScrollRequest).actionGet();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
client.close()
return results;
}
}
@@ -1,84 +0,0 @@
package com.fangjia.script.util
import org.apache.commons.lang3.SystemUtils
import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.settings.Settings
import org.elasticsearch.common.transport.TransportAddress
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
import java.util.concurrent.TimeUnit
/**
* 功能说明:Elasticsearch工具类
*/
class ElasticsearchHelper {
static getClient(String host, String clusterName = "fangjia"){
Settings.Builder builder = Settings.builder()
.put("cluster.name", clusterName)
.put("client.transport.sniff", false)
.put("client.transport.ping_timeout", "60s")
.put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3")
.put("xpack.security.transport.ssl.enabled", "true")
if(SystemUtils.IS_OS_WINDOWS){
builder
.put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
.put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
}else{
builder
.put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12")
.put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12")
}
builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build()
Settings settings = builder.build()
return new PreBuiltXPackTransportClient(settings)
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
}
/**
* 创建一个ES批处理器
* @param clusterName
* @param host
* @return
*/
static createBulkProcessor( String host, String clusterName){
TransportClient client = getClient(host, clusterName)
BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {
@Override
void beforeBulk(long executionId, BulkRequest request) {
}
@Override
void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
}
@Override
void afterBulk(long executionId, BulkRequest request, Throwable failure) {
failure.printStackTrace()
}
}).setBulkActions(10000).setConcurrentRequests(20).build()
return bulkProcessor
}
/**
* 关闭一个ES批处理器
* @param
* @return
*/
static closeBulkProcessor(BulkProcessor bulkProcessor){
bulkProcessor.awaitClose(10, TimeUnit.MINUTES)
bulkProcessor.flush()
bulkProcessor.close()
}
}
@@ -1,19 +1,18 @@
package com.fangjia.script.vipsite
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.ElasticUtils
import com.mongodb.DBObject
import org.elasticsearch.action.search.SearchRequestBuilder
import org.elasticsearch.action.search.SearchResponse
import org.elasticsearch.action.search.SearchType
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit
import org.elasticsearch.search.SearchHits
try {
TransportClient transportClient = ElasticsearchHelper.getClient("192.168.0.206");
TransportClient transportClient = ElasticUtils.getClient("192.168.0.206");
QueryBuilder dateRange_avg_price = QueryBuilders.rangeQuery("avg_price").lte(0);
QueryBuilder dateRange_total_price = QueryBuilders.rangeQuery("total_price").lte(0);
@@ -1,6 +1,6 @@
package com.fangjia.script.weilei
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.MongoUtils
import com.mongodb.Bytes
import com.mongodb.DBCollection
@@ -8,7 +8,7 @@ import com.mongodb.DBCursor
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -16,7 +16,7 @@ import java.sql.SQLException
import java.sql.Statement
import java.text.SimpleDateFormat
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties();
@@ -1,31 +1,16 @@
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.CommandResult
import com.mongodb.DB
import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*;
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
@@ -43,7 +28,7 @@ cityList.each { def city ->
}
})
roots.each { def root ->
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,23 +1,12 @@
package com.fangjia.script.wenyan.address
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.util.QQKeyPool
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -44,7 +33,7 @@ import java.sql.Statement
// data << (k.split("_") as List)
//}
//ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û")
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,20 @@
package com.fangjia.script.wenyan.avgPrice
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
def findEstateType(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math3.ml.clustering.CentroidCluster
import org.apache.commons.math3.ml.clustering.DoublePoint
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017)
def findHouse(BasicDBObject district){
@@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){
sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ")
sqlBuffer.append(" ORDER BY p_date desc")
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s";
dataSource.setUrl(address);
Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource();
EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,20 +1,12 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
//
//def data = [["城市", "区域", "小区名", "2018", "全部"]]
//cityList = ["上海"]
@@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq
//HttpRequest httpRequest = new HttpRequest()
//println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8")
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config")
AVERAGER_CONFIG.find().each {Document doc->
@@ -0,0 +1,82 @@
package com.fangjia.script.wenyan.temp
import cc.blynk.clickhouse.copy.CopyManager
import cc.blynk.clickhouse.copy.CopyManagerFactory
import com.fangjia.core.util.DateUtil
import net.sf.json.JSONNull
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import ru.yandex.clickhouse.ClickHouseConnection
import ru.yandex.clickhouse.ClickHouseDriver
import ru.yandex.clickhouse.settings.ClickHouseProperties
import java.sql.PreparedStatement
import java.sql.Timestamp
//ClickHouseDriver clickHouseDriver = new ClickHouseDriver()
//ClickHouseProperties clickHouseProperties = new ClickHouseProperties()
//clickHouseProperties.connectionTimeout = 1000 * 3600 * 24
//clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24
//clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24
//clickHouseProperties.user = "default"
//clickHouseProperties.password = "goojia123456"
//ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties)
//CopyManager copyManager = CopyManagerFactory.create(connection)
//copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv");
//PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
//def cols = ["id", "city", "region", "name", "district_name", "address", "room", "hall", "toilet", "area", "avg_price", "total_price ", "floor_info", "height", "floor", "direction", "fitment", "p_date", "publish_date", "c_date", "m_date"]
//int count = 0
//new File("D:\\½¨ÐвâÊÔÊý¾Ý").eachFile { def file ->
// file.eachLine("UTF-8", { def line ->
// String id = line.substring(0, line.indexOf("\t"))
// String jsonStr = line.substring(line.indexOf("\t") + 1)
// count++
// JSONObject json = JSONObject.fromObject(jsonStr)
// json.put("id", id)
// for (int i = 0; i < cols.size(); i++) {
// if (cols[i].toString().endsWith("date") && "p_date" != cols[i]) {
// if ("c_date" == cols[i]) {
// if (json.get("m_date") instanceof Long) {
// sth.setTimestamp(i + 1, new Timestamp(json.get("m_date")))
// } else {
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get("m_date"))).getTime()))
//
// }
// } else {
// if (json.get(cols[i]) instanceof String) {
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get(cols[i]))).getTime()))
// } else {
// sth.setTimestamp(i + 1, new Timestamp(json.get(cols[i])))
//
// }
// }
// } else {
// if (json.get(cols[i]) instanceof JSONNull) {
// sth.setObject(i + 1, null)
// } else {
// sth.setObject(i + 1, json.get(cols[i]))
// }
// }
// }
// sth.addBatch()
// if (count % 30000 == 0) {
// sth.executeBatch()
// println(new Date().format("HH:mm:ss " + count))
// }
// })
//}
//sth.execute()
def ids = new HashSet()
for(int i =0 ; i < 10000 * 10000; i++){
if(i%10000 == 0){
println(i)
}
def id = new ObjectId().toString()
if(ids.contains(id)){
println(id)
}else{
ids << id
}
}
@@ -1,17 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\建行深开数据统计.xlsx",{List item->
String city = RegionData.formatCity(item["A"])
if(city){
item["G"] = DbUtils.seaweed.count(DbUtils.buildSeaweedQuery(RegionData.formatCity(item["A"])))
}else{
item["G"] = null
def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]]
["ID", "北京", "重庆", "哈尔滨"].each {def city->
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district->
def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")]
println(item)
data << item
}
})
}
ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据")
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20201101", "yyyyMMdd")
def endDate = DateUtil.parseDate("20201201", "yyyyMMdd")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
@@ -1,19 +1,13 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection
import org.apache.commons.lang.StringUtils
import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -61,7 +55,7 @@ import java.sql.Statement
//println(bigMarkupCount)
def countHouses(String city, String region, String name){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,28 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.bson.Document
import com.mongodb.client.MongoCollection
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def poiData = Collections.synchronizedList([])
//
//ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item ->
@@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([])
// println(item)
//},1)
def findHouseCount(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -0,0 +1,42 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if(!result.buildYear || !result.avgPrice){
return
}
def params = JSONObject.fromObject(doc.parameter_map)
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << key
} else {
println("已存在")
}
}
startDate = nextDate
}
ExcelUtils.write("D:\\四川锦城${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
@@ -1,11 +1,17 @@
import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
//import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
@@ -132,4 +138,70 @@ import org.apache.commons.lang3.StringUtils
// item["E"] = RegionData.formatRegion(item["B"], item["C"])
// println(item)
//})
CommonUtils.reFitAverage("常州","新北","龙城花园", "普通住宅")
//def border = [13511506.7,3639168.61,13511516.8,3639404.86,13511515.27,3639416.92,13511738.21,3639384.94,13512007.72,3639336.35,13511984.92,3639074.96,13511722.75,3639121.08,13511506.7,3639168.61]
//def coords = []
//for(int i=0; i < border.size() - 1; i=i+2){
// coords << "${border[i]},${border[i+1]}"
//}
//def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
//println(json)
//def points = []
//for(JSONObject it : json.result){
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// points << "${geo.lng},${geo.lat}"
// println(" new AMap.LngLat(${geo.lng},${geo.lat}),")
//}
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
def coords = []
def baidu_points = []
def gaode_points = []
for(int i=0; i < border.size() - 1; i=i+2){
coords << "${border[i]},${border[i+1]}"
if(coords.size() > 50){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
coords.clear()
}
}
if(coords.size() > 0){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
}
dbo.format_city = RegionData.formatCity(dbo.city)
dbo.baidu_border = baidu_points.join(";")
dbo.gaode_border = gaode_points.join(";")
BAIDUMAP_BORDER_COLL.save(dbo)
println(dbo)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
// if(seaweed){
// item["A"] = seaweed._id.toString()
// def names = [item['D']]
// if(seaweed.alias){
// names.addAll(seaweed.alias)
// }
// names.unique()
// if(seaweed.lng){
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
// }
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
// if(one){
// item["I"] = one.baidu_border
// }
// println(item)
// }
//})
@@ -669,7 +669,6 @@ def findRentInfoByRoom(String city, String region){
def findRent(def city, def region, def name){
DBCollection DISTRICT_STAT_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
def query = new BasicDBObject()
query.city = city
query.region = region
@@ -14,6 +14,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.client.MongoCollection
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.apache.commons.lang3.StringUtils
import org.dom4j.Document
@@ -204,52 +205,71 @@ import org.dom4j.io.SAXReader
//// }
//// }
//})
DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
statMap = [:]
def s = new HashSet()
int count = 0
def data = [["城市", "地址"]]
def query = new BasicDBObject("user_id", "1007751").append("c_date", new BasicDBObject('$gte', Date.parse("yyyyMMdd", "20210119")))
println JSONObject.fromObject(query)
println COMMAND_COLL.count(query)
COMMAND_COLL.find(query).batchSize(10000).each { BasicDBObject log ->
count++
// def city = log.parameter_map.city
// def stat = statMap.get(city, [:])
// stat["调用次数"] = stat.get("调用次数", 0) + 1
if (log.res_map && log.res_map.result && log.res_map.result.totalPrice) {
// stat["估价成功次数"] = stat.get("估价成功次数", 0) + 1
// if(log.res_map.result.calcType == 1){
// stat["有样本"] = stat.get("有样本", 0) + 1
// }else{
// stat["无样本"] = stat.get("无样本", 0) + 1
//DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
//statMap = [:]
//def s = new HashSet()
//int count = 0
//def data = [["城市", "地址"]]
//def query = new BasicDBObject("user_id", "1007751").append("c_date", new BasicDBObject('$gte', Date.parse("yyyyMMdd", "20210119")))
//println JSONObject.fromObject(query)
//println COMMAND_COLL.count(query)
//COMMAND_COLL.find(query).batchSize(10000).each { BasicDBObject log ->
// count++
//// def city = log.parameter_map.city
//// def stat = statMap.get(city, [:])
//// stat["调用次数"] = stat.get("调用次数", 0) + 1
// if (log.res_map && log.res_map.result && log.res_map.result.totalPrice) {
//// stat["估价成功次数"] = stat.get("估价成功次数", 0) + 1
//// if(log.res_map.result.calcType == 1){
//// stat["有样本"] = stat.get("有样本", 0) + 1
//// }else{
//// stat["无样本"] = stat.get("无样本", 0) + 1
//// }
//// stat["总价"] = stat.get("总价", 0l) + log.res_map.result.totalPrice
//// stat["总面积"] = stat.get("总面积", 0d) + log.parameter_map.size.toDouble()
//
// } else {
// if (log.res_map.msg.toString().contains("该小区不存在")) {
// println( [log.parameter_map.city, log.parameter_map.address])
// data << [log.parameter_map.city, log.parameter_map.address]
// }
// stat["总价"] = stat.get("总价", 0l) + log.res_map.result.totalPrice
// stat["总面积"] = stat.get("总面积", 0d) + log.parameter_map.size.toDouble()
} else {
if (log.res_map.msg.toString().contains("该小区不存在")) {
println( [log.parameter_map.city, log.parameter_map.address])
data << [log.parameter_map.city, log.parameter_map.address]
}
// stat["估价失败次数"] = stat.get("估价失败次数", 0) + 1
// stat["失败原因:" + log.res_map.msg] = stat.get("失败原因:" + log.res_map.msg, 0) + 1
// s << "失败原因:" + log.res_map.msg
}
if (count % 10000 == 0) {
println(count)
}
}
data = data.unique()
ExcelUtils.write("D:\\未匹配地址.xlsx", data, "结果")
//key = ["调用次数", "估价成功次数", "有样本", "无样本", "总价", "总面积", "估价失败次数"]
//key.addAll(s)
//print(key.join("\t"))
//[ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ].each {def city->
// stat = statMap.get(city)
// con = [city]
// for( def k : key){
// con << stat[k]
//// stat["估价失败次数"] = stat.get("估价失败次数", 0) + 1
//// stat["失败原因:" + log.res_map.msg] = stat.get("失败原因:" + log.res_map.msg, 0) + 1
//// s << "失败原因:" + log.res_map.msg
// }
// println con.join("\t")
//}
// if (count % 10000 == 0) {
// println(count)
// }
//}
//data = data.unique()
//ExcelUtils.write("D:\\未匹配地址.xlsx", data, "结果")
////key = ["调用次数", "估价成功次数", "有样本", "无样本", "总价", "总面积", "估价失败次数"]
////key.addAll(s)
////print(key.join("\t"))
////[ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ].each {def city->
//// stat = statMap.get(city)
//// con = [city]
//// for( def k : key){
//// con << stat[k]
//// }
//// println con.join("\t")
////}
//
//DBCollection OPEN_URI_LOG_42 = MongoUtils.getCollectionByName("business", "open_uri_log", "log.mongo.db.fangjia.com", 52002)
//def data = [["城市", "地址", "省", "市", "区", "开发区","乡镇", "路", "路号", "小区名", "楼栋号", "单元号", "室号", "标准化地址"]]
//OPEN_URI_LOG_42.find(new BasicDBObject("user_id", "1007756")).each {BasicDBObject dbo->
// data << [dbo.parameter_map.city, dbo.parameter_map.address, dbo.res_map.result.province, dbo.res_map.result.city, dbo.res_map.result.district, dbo.res_map.result.developmentZone, dbo.res_map.result.township, dbo.res_map.result.street,dbo.res_map.result.streetNumber,dbo.res_map.result.name,dbo.res_map.result.buildingNumber,dbo.res_map.result.unitNumber,dbo.res_map.result.roomNumber,dbo.res_map.result.formattedAddress]
//}
//ExcelUtils.write("D:\\建行测试地址.xlsx", data, "结果")
DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
def query = new BasicDBObject()
query.cat = "region"
query.city= "上海"
query.weekend = Date.parse("yyyyMMdd", "20210228")
def doc = []
def prices = []
SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price))
prices << it.avg_price
}
println(JSONArray.fromObject(doc).toString())
@@ -1,19 +1,8 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\下载记录20200818-结果 - 副本.xlsx",{List item->
@@ -1,23 +1,9 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.mongodb.DBCollection
//
//ExcelUtils.multiThreadingModify("D:\\数据\\建行学校数据\\上海学校_0811.xls",{List item->
@@ -5,13 +5,12 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
//def data = []
//data << ["城市", "区域", "小区名", "近一月", "近三月", "近6月", "近一年", "户数"]
//DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("上海")).each {BasicDBObject district->
@@ -36,7 +35,7 @@ import java.sql.Statement
//ExcelUtils.write("D:\\上海成交统计.xlsx", data, "成交数据")
def find_rent_house_prices(String city, String region, String name){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -80,7 +79,7 @@ def find_rent_house_prices(String city, String region, String name){
}
ExcelUtils.multiThreadingModify("D:\\上海成交统计.xlsx", {List item->
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,13 +1,12 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.script.util.*
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import com.mongodb.DBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
@@ -16,7 +15,7 @@ import java.sql.Statement
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
def String hasElevator(BasicDBObject seaweed){
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
@@ -1,40 +1,18 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.script.util.ElasticUtils
import org.elasticsearch.index.query.QueryBuilders
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
//ElasticUtils.executeSqlQuery("select * from seawater where p_date > 20210407 limit 10",{ Map row->
// println(row.name)
//}, "192.168.0.216")
ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\新增小区.xlsx", 0, 0).each {List item->
def keys = new BasicDBObject()
keys.city = 1
keys.region = 1
keys.name = 1
keys.estate_type2 = 1
def query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
if(!DbUtils.findDistrictStat(dbo, new Date())){
CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
println(dbo)
}
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
// println(dbo)
}
}
//["佛山","济南","北京","广州","深圳"].each {def city->
// def keys = new BasicDBObject()
// keys.city = 1
// keys.region = 1
// keys.name = 1
// keys.estate_type2 = 1
// def query = DbUtils.buildSeaweedQuery(city)
// query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
// DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
// println(dbo)
// }
//ElasticUtils.executeSimpleQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "ÐÂãþÈý´å")).each {
// println(it)
//}
ElasticUtils.executeQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "ÐÂãþÈý´å"),{
println(it)
})
@@ -1,20 +1,14 @@
package com.fangjia.script.yinjihaun.price
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
JdbcDataSource dataSource = new JdbcDataSource()
EsDataSource dataSource = new EsDataSource()
dataSource.setUrl(address)
Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer")