Merge branch 'master' of http://gitlab.fangjia.com/dev0/main
This commit is contained in:
@@ -17,8 +17,8 @@
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<encoding>GBK</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
@@ -111,6 +111,11 @@
|
||||
<artifactId>fangjia-datastore-driver</artifactId>
|
||||
<version>3.9.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>cc.blynk.clickhouse</groupId>
|
||||
<artifactId>clickhouse4j</artifactId>
|
||||
<version>1.4.4</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>redis.clients</groupId>
|
||||
<artifactId>jedis</artifactId>
|
||||
@@ -174,7 +179,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.4.1</version>
|
||||
<version>4.5.10</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.ant</groupId>
|
||||
@@ -310,16 +315,6 @@
|
||||
<artifactId>metadata-extractor</artifactId>
|
||||
<version>2.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.http</groupId>
|
||||
<artifactId>httpasyncclient</artifactId>
|
||||
<version>4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.http</groupId>
|
||||
<artifactId>httpcore-nio</artifactId>
|
||||
<version>4.1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.sanselan</groupId>
|
||||
<artifactId>sanselan</artifactId>
|
||||
@@ -443,23 +438,18 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>x-pack-transport</artifactId>
|
||||
<version>6.4.2</version>
|
||||
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
||||
<version>7.11.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.plugin</groupId>
|
||||
<artifactId>x-pack-sql-jdbc</artifactId>
|
||||
<version>6.4.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>transport</artifactId>
|
||||
<version>6.4.2</version>
|
||||
<version>7.11.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-core</artifactId>
|
||||
<version>7.4.0</version>
|
||||
<version>8.7.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
|
||||
+2
-2
@@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
|
||||
|
||||
def findHouseMaxArea(String city, String region, String name, List alias) {
|
||||
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24)
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-3
@@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection
|
||||
import com.mongodb.client.model.UpdateOptions
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it->
|
||||
names << it.name
|
||||
}
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
|
||||
def findHouseCount(String city, String region, String name, List alias, int months) {
|
||||
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months)
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+5
-9
@@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement;
|
||||
import java.sql.Statement
|
||||
|
||||
/**
|
||||
*
|
||||
* 历史均价计算
|
||||
@@ -27,7 +23,7 @@ import java.sql.Statement;
|
||||
*/
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class HistoricalAverageRevaluationTask extends BaseBatch{
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
|
||||
public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat')
|
||||
|
||||
+2
-13
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.avgprice
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
@@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.Bytes
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement;
|
||||
import com.mongodb.*
|
||||
|
||||
/**
|
||||
* 无样本小区历史遗漏数据补充
|
||||
|
||||
+2
-2
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -4,13 +4,13 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class AddEsHouseField {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+2
-3
@@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
@@ -63,7 +62,7 @@ class AddThirdPriceDistrict {
|
||||
def province = null
|
||||
def resAddrMap = [:]
|
||||
def filterEstateTypeMap = [:]
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.esms.common.util.CommonUtil
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.huaban.analysis.jieba.JiebaSegmenter
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -485,7 +485,7 @@ class CrawlDataMatch {
|
||||
}
|
||||
|
||||
def findAllAddress(def cityName){
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -1,24 +1,19 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.SQLException
|
||||
import java.sql.Statement
|
||||
|
||||
class EsHouseStat {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class EsSearchUtils {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
@@ -20,7 +19,7 @@ class FZDataExport {
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+2
-3
@@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{
|
||||
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -3,14 +3,14 @@ package com.fangjia.script.gj
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class RoomStat {
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -1,30 +1,25 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.idsQuery
|
||||
|
||||
class TradeDataMatch {
|
||||
|
||||
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-4
File diff suppressed because one or more lines are too long
+7
-15
@@ -1,24 +1,17 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.SimpleDateFormat
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -26,12 +19,11 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
|
||||
class HouseExport {
|
||||
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
@@ -83,11 +75,11 @@ class HouseExport {
|
||||
// exportRentCSV(city, districts, 20200201, 20210301, 2020);
|
||||
}else{
|
||||
|
||||
exportRentCSV(city, districts, 20170801, 20181001, 2017)
|
||||
exportRentCSV(city, districts, 20200201, 20210301, 2020);
|
||||
// exportRentCSV(city, districts, 20170801, 20181001, 2017)
|
||||
// exportRentCSV(city, districts, 20200201, 20210301, 2020);
|
||||
|
||||
exportSellCSV(city, districts, 20170801, 20181001, 2017);
|
||||
exportSellCSV(city, districts, 20200201, 20210301, 2020);
|
||||
// exportSellCSV(city, districts, 20200201, 20210301, 2020);
|
||||
|
||||
}
|
||||
|
||||
|
||||
+2
-4
@@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
class ExportData {
|
||||
|
||||
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+3
-8
@@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,7 +22,7 @@ class DistrictWash {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+17
-27
@@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.DecimalFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils;
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
import com.fangjia.script.util.ExcelParser
|
||||
import com.fangjia.script.common.DataHandleUtils;
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.script.util.MongoUtils;
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.Map.Entry;
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.DecimalFormat
|
||||
import java.util.Map.Entry
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CZBlankDataExport {
|
||||
|
||||
@@ -62,7 +52,7 @@ class CZBlankDataExport {
|
||||
def directionMap = [1:"ÄÏ",2:"±±",3:"Äϱ±",4:"¶«",5:"Î÷",6:"¶«Î÷",7:"¶«ÄÏ",8:"Î÷ÄÏ",9:"¶«±±",10:"Î÷±±"]
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
-1
@@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
/**
|
||||
* 江南银行线上数据交付
|
||||
|
||||
+2
-17
@@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.CrawlCommandConsumerBatch
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.BatchUpdateException
|
||||
import java.sql.ResultSet
|
||||
import java.text.NumberFormat
|
||||
import java.util.Map.Entry
|
||||
import java.util.concurrent.BlockingQueue
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.LinkedBlockingDeque
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
import java.util.concurrent.ThreadPoolExecutor
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@BatchOwner(Watcher.WeiLei)
|
||||
class HousingOverviewExportBatch extends BaseBatch {
|
||||
|
||||
static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>())
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+7
-10
@@ -1,11 +1,10 @@
|
||||
package com.fangjia.script.jusfoun
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import groovy.sql.GroovyRowResult
|
||||
@@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import redis.clients.jedis.JedisPool
|
||||
import redis.clients.jedis.JedisPoolConfig
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.*
|
||||
import java.text.DateFormat
|
||||
@@ -56,7 +53,7 @@ class JusfounCommon {
|
||||
public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
|
||||
public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
|
||||
|
||||
private static JdbcDataSource dataSource = null
|
||||
private static EsDataSource dataSource = null
|
||||
|
||||
private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>();
|
||||
private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>();
|
||||
@@ -390,7 +387,7 @@ class JusfounCommon {
|
||||
return DataUtil.hanyuQuanpin(hanzi, '')
|
||||
}
|
||||
|
||||
static JdbcDataSource getESDataSource() {
|
||||
static EsDataSource getESDataSource() {
|
||||
if (dataSource != null) {
|
||||
return dataSource;
|
||||
} else {
|
||||
@@ -399,7 +396,7 @@ class JusfounCommon {
|
||||
return dataSource;
|
||||
}
|
||||
try {
|
||||
dataSource = new JdbcDataSource();
|
||||
dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties();
|
||||
@@ -625,13 +622,13 @@ class JusfounCommon {
|
||||
String scrollID = null) {
|
||||
try {
|
||||
if (transportClient == null) {
|
||||
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
}
|
||||
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace()
|
||||
try {
|
||||
transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
return getEsDataOnce(transportClient, city, cat, startDate, endDate, INDEX, scrollID);
|
||||
} catch (Exception e1) {
|
||||
e1.printStackTrace()
|
||||
|
||||
-8
@@ -8,21 +8,13 @@ import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import java.sql.BatchUpdateException
|
||||
import java.util.concurrent.LinkedBlockingDeque
|
||||
import java.util.concurrent.ThreadPoolExecutor
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
@BatchOwner(Watcher.WangWenYan)
|
||||
class RentHouseExportBatch extends BaseBatch {
|
||||
|
||||
+2
-2
@@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
def List find_rent_house_prices(String city, String region, String name, Date start, Date end){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+5
-8
@@ -1,20 +1,17 @@
|
||||
package com.fangjia.script.log
|
||||
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class MongoDBLogDisMonitorBatch extends BaseBatch {
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
|
||||
+6
-8
@@ -1,20 +1,18 @@
|
||||
package com.fangjia.script.log
|
||||
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class MongoDBLogMonitorBatch extends BaseBatch {
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
|
||||
+3
-2
@@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
@@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch {
|
||||
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final JdbcDataSource dataSource = new JdbcDataSource()
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-4
@@ -5,7 +5,7 @@ import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
@@ -55,7 +55,6 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
Map userInfo = getUserInfo()
|
||||
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
|
||||
def endDate = DateUtil.truncateDay(new Date())
|
||||
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
|
||||
@@ -64,7 +63,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
long count = 0
|
||||
long total = cursor.count()
|
||||
print("total="+total)
|
||||
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia")
|
||||
BulkProcessor bulkProcessor = ElasticUtils.createBulkProcessor("seawater.es.db.fangjia.com")
|
||||
for(BasicDBObject log : cursor){
|
||||
Map<String,Object> parameter_map = (Map) log.get("parameter_map")
|
||||
if(parameter_map == null) {
|
||||
@@ -128,7 +127,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
cmd.put("user_name", user.user_name)
|
||||
cmd.put("company_name", user.company)
|
||||
}
|
||||
bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd))
|
||||
bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd))
|
||||
println(cmd)
|
||||
count++
|
||||
if(count % 100 == 0){
|
||||
|
||||
+11
-37
@@ -1,50 +1,24 @@
|
||||
package com.fangjia.script.trade
|
||||
///µ¼Èë¼Û¸ñÖ¸Êý
|
||||
import com.fangjia.script.BaseBatch;
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
|
||||
///导入价格指数
|
||||
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
||||
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.poi.xssf.usermodel.XSSFCell
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.NumberFormat;
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.fangjia.core.util.DBIterator
|
||||
|
||||
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils;
|
||||
import com.ibm.icu.text.SimpleDateFormat;
|
||||
import com.fangjia.storehouse.DBIterator
|
||||
import com.fangjia.storehouse.DBIterator;
|
||||
import com.fangjia.storehouse.StorehouseDB;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ImportIndexStats extends BaseBatch{
|
||||
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
|
||||
@@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
public static final def dingjiYear = 2016;
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final JdbcDataSource dataSource = new JdbcDataSource()
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-8
@@ -1,17 +1,12 @@
|
||||
package com.fangjia.script.trade
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
@@ -22,7 +17,7 @@ class TradeDealPriceStat {
|
||||
|
||||
|
||||
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ package com.fangjia.script.usahouse
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
|
||||
|
||||
class AddressMatch {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+1
-4
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.usahouse
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.BaseBatch
|
||||
@@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSON
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
+174
-156
@@ -1,122 +1,170 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.commons.lang3.SystemUtils
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.action.index.IndexRequest
|
||||
import org.elasticsearch.action.search.ClearScrollRequest
|
||||
import org.elasticsearch.action.search.SearchRequest
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchType
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.settings.Settings
|
||||
import org.elasticsearch.common.transport.TransportAddress
|
||||
import org.elasticsearch.common.unit.ByteSizeUnit
|
||||
import org.elasticsearch.common.unit.ByteSizeValue
|
||||
import org.elasticsearch.action.search.SearchScrollRequest
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.Scroll
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.sort.FieldSortBuilder
|
||||
import org.elasticsearch.search.sort.SortOrder
|
||||
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
|
||||
import org.elasticsearch.search.builder.SearchSourceBuilder
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticUtils {
|
||||
|
||||
private static TransportClient client = null;
|
||||
|
||||
public static TransportClient getClient(def host){
|
||||
if(client != null){
|
||||
return client;
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeSqlQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
|
||||
def certPath = ""
|
||||
if(SystemUtils.IS_OS_WINDOWS){
|
||||
certPath= "D:\\goojiaconfig\\seawater\\conf\\elastic-certificates.p12"
|
||||
}else{
|
||||
certPath = "/opt/elasticmongo/elastic-certificates.p12"
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
Settings.Builder builder = Settings.builder()
|
||||
.put("cluster.name", "fangjia")
|
||||
.put("client.transport.sniff", true)
|
||||
|
||||
builder.put("xpack.security.user", "developer" + ":" + "5e8edd851d2fdfbd7415232c67367cc3")
|
||||
.put("xpack.security.transport.ssl.enabled", "true")
|
||||
.put("xpack.security.transport.ssl.truststore.path", certPath)
|
||||
.put("xpack.security.transport.ssl.keystore.path", certPath)
|
||||
.put("xpack.security.transport.ssl.verification_mode", "certificate")
|
||||
|
||||
Settings settings = builder.build();
|
||||
client = new PreBuiltXPackTransportClient(settings)
|
||||
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
|
||||
return client
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure, String type= "_doc"){
|
||||
|
||||
client = getClient(host);
|
||||
|
||||
BulkProcessor bulkProcessor = BulkProcessor.builder(
|
||||
client,
|
||||
new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId,
|
||||
BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId,
|
||||
BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId,
|
||||
BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
})
|
||||
.setBulkActions(10000)
|
||||
.setBulkSize(new ByteSizeValue(5, ByteSizeUnit.MB))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(5))
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.exponentialBackoff(TimeValue.timeValueMillis(100), 10))
|
||||
.build()
|
||||
|
||||
SearchResponse scrollResp = null;
|
||||
int errorCount = 0;
|
||||
def hits ;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
try{
|
||||
scrollResp = client.prepareSearch(indexName)
|
||||
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
|
||||
.setScroll(new TimeValue(600000))
|
||||
.setQuery(qb)
|
||||
.setSize(1000).get()
|
||||
hits = scrollResp.getHits().getHits()
|
||||
stop = true;
|
||||
}catch(Exception e){
|
||||
errorCount++;
|
||||
if(errorCount>11){
|
||||
stop = true;
|
||||
}
|
||||
println "错误:${errorCount} "+e.getMessage()
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 60);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor(String host, RestHighLevelClient client = null) {
|
||||
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
};
|
||||
if(!client){
|
||||
client = getClient(host);
|
||||
}
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
if(hits){
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
|
||||
for (int k=0;k<hits.size();k++) {
|
||||
final SearchHit hit = hits[k];
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
static executeQuery(String host, String indexName, QueryBuilder qb, Closure closure){
|
||||
|
||||
RestHighLevelClient client = getClient(host)
|
||||
BulkProcessor bulkProcessor = createBulkProcessor(host, client)
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def nThread = Runtime.getRuntime().availableProcessors() * 5
|
||||
ExecutorService executorService = Executors.newFixedThreadPool(nThread)
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
final SearchHit hit = searchHits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
@@ -124,96 +172,66 @@ class ElasticUtils {
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
bulkProcessor.add(new IndexRequest(indexName).id(hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
// println(hits)
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
|
||||
while(scrollResp.getHits().getHits().length != 0){
|
||||
hits = scrollResp.getHits().getHits();
|
||||
for (int k=0;k<hits.size();k++) {
|
||||
final SearchHit hit = hits[k];
|
||||
executorService.submit(new Runnable() {
|
||||
@Override
|
||||
void run() {
|
||||
try{
|
||||
if(closure.getMaximumNumberOfParameters() == 1){
|
||||
def update = closure.call(hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}else if(closure.getMaximumNumberOfParameters() == 2){
|
||||
def update = closure.call(hit.getId(), hit.getSourceAsMap())
|
||||
if(update){
|
||||
bulkProcessor.add(new IndexRequest(indexName, type, hit.getId()).source(hit.getSourceAsMap()))
|
||||
}
|
||||
}
|
||||
}catch (Exception e){
|
||||
// println(hits)
|
||||
e.printStackTrace()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(new TimeValue(60000)).execute().actionGet()
|
||||
}
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
executorService.shutdown()
|
||||
executorService.awaitTermination(1, TimeUnit.DAYS)
|
||||
bulkProcessor.close()
|
||||
bulkProcessor.awaitClose(1, TimeUnit.DAYS)
|
||||
client.close()
|
||||
}
|
||||
|
||||
|
||||
static def executeSimpleQuery(String host, String indexName, QueryBuilder qb){
|
||||
client = getClient(host);
|
||||
SearchResponse scrollResp = null;
|
||||
int errorCount = 0;
|
||||
def hits ;
|
||||
def stop = false;
|
||||
while(!stop){
|
||||
try{
|
||||
scrollResp = client.prepareSearch(indexName)
|
||||
.addSort(FieldSortBuilder.DOC_FIELD_NAME, SortOrder.ASC)
|
||||
.setScroll(new TimeValue(600000))
|
||||
.setQuery(qb)
|
||||
.setSize(10000)
|
||||
.setScroll(TimeValue.timeValueMinutes(1))//设置 search context 维护1分钟的有效期
|
||||
.get()
|
||||
|
||||
stop = true;
|
||||
}catch(Exception e){
|
||||
errorCount++;
|
||||
if(errorCount>11){
|
||||
stop = true;
|
||||
}
|
||||
println "错误:${errorCount} "+e.getMessage()
|
||||
}
|
||||
}
|
||||
final Scroll scroll = new Scroll(TimeValue.timeValueMinutes(60L))
|
||||
RestHighLevelClient client = getClient(host)
|
||||
SearchRequest searchRequest = new SearchRequest(indexName);
|
||||
searchRequest.scroll(scroll);
|
||||
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
|
||||
searchSourceBuilder.query(qb);
|
||||
searchRequest.source(searchSourceBuilder);
|
||||
SearchResponse searchResponse = client.search(searchRequest, RequestOptions.DEFAULT);
|
||||
String scrollId = searchResponse.getScrollId();
|
||||
SearchHit[] searchHits = searchResponse.getHits().getHits();
|
||||
def results = []
|
||||
while(scrollResp.getHits() && scrollResp.getHits().getHits().length != 0){
|
||||
for (SearchHit searchHit : scrollResp.getHits().getHits()) {
|
||||
while (searchHits != null && searchHits.length > 0) {
|
||||
for (int k=0; k < searchHits.size(); k++) {
|
||||
SearchHit searchHit = searchHits[k]
|
||||
searchHit.getSourceAsMap().put("id", searchHit.getId());
|
||||
results << searchHit.getSourceAsMap()
|
||||
}
|
||||
scrollResp = client.prepareSearchScroll(scrollResp.getScrollId()).setScroll(TimeValue.timeValueMinutes(1)).execute().actionGet();
|
||||
//当searchHits的数组为空的时候结束循环,至此数据全部读取完毕
|
||||
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
|
||||
scrollRequest.scroll(scroll);
|
||||
searchResponse = client.scroll(scrollRequest, RequestOptions.DEFAULT);
|
||||
scrollId = searchResponse.getScrollId();
|
||||
searchHits = searchResponse.getHits().getHits();
|
||||
}
|
||||
//删除scroll
|
||||
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
|
||||
clearScrollRequest.addScrollId(scrollResp.getScrollId());
|
||||
client.clearScroll(clearScrollRequest).actionGet();
|
||||
clearScrollRequest.addScrollId(scrollId);
|
||||
client.clearScroll(clearScrollRequest, RequestOptions.DEFAULT);
|
||||
client.close()
|
||||
return results;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
-84
@@ -1,84 +0,0 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.commons.lang3.SystemUtils
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.settings.Settings
|
||||
import org.elasticsearch.common.transport.TransportAddress
|
||||
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
|
||||
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticsearchHelper {
|
||||
|
||||
|
||||
static getClient(String host, String clusterName = "fangjia"){
|
||||
Settings.Builder builder = Settings.builder()
|
||||
.put("cluster.name", clusterName)
|
||||
.put("client.transport.sniff", false)
|
||||
.put("client.transport.ping_timeout", "60s")
|
||||
.put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3")
|
||||
.put("xpack.security.transport.ssl.enabled", "true")
|
||||
if(SystemUtils.IS_OS_WINDOWS){
|
||||
builder
|
||||
.put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
|
||||
.put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
|
||||
}else{
|
||||
builder
|
||||
.put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12")
|
||||
.put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12")
|
||||
}
|
||||
builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build()
|
||||
Settings settings = builder.build()
|
||||
return new PreBuiltXPackTransportClient(settings)
|
||||
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param clusterName
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor( String host, String clusterName){
|
||||
TransportClient client = getClient(host, clusterName)
|
||||
BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request, Throwable failure) {
|
||||
failure.printStackTrace()
|
||||
}
|
||||
}).setBulkActions(10000).setConcurrentRequests(20).build()
|
||||
|
||||
return bulkProcessor
|
||||
}
|
||||
|
||||
/**
|
||||
* 关闭一个ES批处理器
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor){
|
||||
|
||||
bulkProcessor.awaitClose(10, TimeUnit.MINUTES)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
+2
-3
@@ -1,19 +1,18 @@
|
||||
package com.fangjia.script.vipsite
|
||||
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.action.search.SearchRequestBuilder
|
||||
import org.elasticsearch.action.search.SearchResponse
|
||||
import org.elasticsearch.action.search.SearchType
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.search.SearchHits
|
||||
|
||||
try {
|
||||
TransportClient transportClient = ElasticsearchHelper.getClient("192.168.0.206");
|
||||
TransportClient transportClient = ElasticUtils.getClient("192.168.0.206");
|
||||
|
||||
QueryBuilder dateRange_avg_price = QueryBuilders.rangeQuery("avg_price").lte(0);
|
||||
QueryBuilder dateRange_total_price = QueryBuilders.rangeQuery("total_price").lte(0);
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.weilei
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
@@ -8,7 +8,7 @@ import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -16,7 +16,7 @@ import java.sql.SQLException
|
||||
import java.sql.Statement
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties();
|
||||
|
||||
@@ -1,31 +1,16 @@
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.CommandResult
|
||||
import com.mongodb.DB
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.*;
|
||||
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import static com.mongodb.client.model.Filters.and
|
||||
import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
|
||||
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
|
||||
@@ -43,7 +28,7 @@ cityList.each { def city ->
|
||||
}
|
||||
})
|
||||
roots.each { def root ->
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-14
@@ -1,23 +1,12 @@
|
||||
package com.fangjia.script.wenyan.address
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.util.QQKeyPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -44,7 +33,7 @@ import java.sql.Statement
|
||||
// data << (k.split("_") as List)
|
||||
//}
|
||||
//ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û")
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-3
@@ -1,20 +1,20 @@
|
||||
package com.fangjia.script.wenyan.avgPrice
|
||||
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
def findEstateType(String city, String region, String name, List alias) {
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-7
@@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math3.ml.clustering.CentroidCluster
|
||||
import org.apache.commons.math3.ml.clustering.DoublePoint
|
||||
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017)
|
||||
|
||||
def findHouse(BasicDBObject district){
|
||||
@@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){
|
||||
sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ")
|
||||
sqlBuffer.append(" ORDER BY p_date desc")
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
package com.fangjia.script.wenyan.seawater
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
package com.fangjia.script.wenyan.seawater
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-14
@@ -1,20 +1,12 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.and
|
||||
import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
//
|
||||
//def data = [["城市", "区域", "小区名", "2018", "全部"]]
|
||||
//cityList = ["上海"]
|
||||
@@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
//HttpRequest httpRequest = new HttpRequest()
|
||||
//println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8")
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config")
|
||||
AVERAGER_CONFIG.find().each {Document doc->
|
||||
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import cc.blynk.clickhouse.copy.CopyManager
|
||||
import cc.blynk.clickhouse.copy.CopyManagerFactory
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import net.sf.json.JSONNull
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.types.ObjectId
|
||||
import ru.yandex.clickhouse.ClickHouseConnection
|
||||
import ru.yandex.clickhouse.ClickHouseDriver
|
||||
import ru.yandex.clickhouse.settings.ClickHouseProperties
|
||||
|
||||
import java.sql.PreparedStatement
|
||||
import java.sql.Timestamp
|
||||
|
||||
//ClickHouseDriver clickHouseDriver = new ClickHouseDriver()
|
||||
//ClickHouseProperties clickHouseProperties = new ClickHouseProperties()
|
||||
//clickHouseProperties.connectionTimeout = 1000 * 3600 * 24
|
||||
//clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24
|
||||
//clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24
|
||||
//clickHouseProperties.user = "default"
|
||||
//clickHouseProperties.password = "goojia123456"
|
||||
//ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties)
|
||||
//CopyManager copyManager = CopyManagerFactory.create(connection)
|
||||
//copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv");
|
||||
|
||||
|
||||
//PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
|
||||
//def cols = ["id", "city", "region", "name", "district_name", "address", "room", "hall", "toilet", "area", "avg_price", "total_price ", "floor_info", "height", "floor", "direction", "fitment", "p_date", "publish_date", "c_date", "m_date"]
|
||||
//int count = 0
|
||||
//new File("D:\\½¨ÐвâÊÔÊý¾Ý").eachFile { def file ->
|
||||
// file.eachLine("UTF-8", { def line ->
|
||||
// String id = line.substring(0, line.indexOf("\t"))
|
||||
// String jsonStr = line.substring(line.indexOf("\t") + 1)
|
||||
// count++
|
||||
// JSONObject json = JSONObject.fromObject(jsonStr)
|
||||
// json.put("id", id)
|
||||
// for (int i = 0; i < cols.size(); i++) {
|
||||
// if (cols[i].toString().endsWith("date") && "p_date" != cols[i]) {
|
||||
// if ("c_date" == cols[i]) {
|
||||
// if (json.get("m_date") instanceof Long) {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(json.get("m_date")))
|
||||
// } else {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get("m_date"))).getTime()))
|
||||
//
|
||||
// }
|
||||
// } else {
|
||||
// if (json.get(cols[i]) instanceof String) {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get(cols[i]))).getTime()))
|
||||
// } else {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(json.get(cols[i])))
|
||||
//
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// if (json.get(cols[i]) instanceof JSONNull) {
|
||||
// sth.setObject(i + 1, null)
|
||||
// } else {
|
||||
// sth.setObject(i + 1, json.get(cols[i]))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// sth.addBatch()
|
||||
// if (count % 30000 == 0) {
|
||||
// sth.executeBatch()
|
||||
// println(new Date().format("HH:mm:ss " + count))
|
||||
// }
|
||||
// })
|
||||
//}
|
||||
//sth.execute()
|
||||
def ids = new HashSet()
|
||||
for(int i =0 ; i < 10000 * 10000; i++){
|
||||
if(i%10000 == 0){
|
||||
println(i)
|
||||
}
|
||||
def id = new ObjectId().toString()
|
||||
if(ids.contains(id)){
|
||||
println(id)
|
||||
}else{
|
||||
ids << id
|
||||
}
|
||||
}
|
||||
+10
-11
@@ -1,17 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\建行深开数据统计.xlsx",{List item->
|
||||
String city = RegionData.formatCity(item["A"])
|
||||
if(city){
|
||||
item["G"] = DbUtils.seaweed.count(DbUtils.buildSeaweedQuery(RegionData.formatCity(item["A"])))
|
||||
}else{
|
||||
item["G"] = null
|
||||
def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]]
|
||||
["ID", "北京", "重庆", "哈尔滨"].each {def city->
|
||||
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district->
|
||||
def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")]
|
||||
println(item)
|
||||
data << item
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据")
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20201101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20201201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+3
-9
@@ -1,19 +1,13 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -61,7 +55,7 @@ import java.sql.Statement
|
||||
//println(bigMarkupCount)
|
||||
|
||||
def countHouses(String city, String region, String name){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-15
@@ -1,28 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
def poiData = Collections.synchronizedList([])
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item ->
|
||||
@@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([])
|
||||
// println(item)
|
||||
//},1)
|
||||
def findHouseCount(String city, String region, String name, List alias) {
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+42
@@ -0,0 +1,42 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.bson.Document
|
||||
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
def hashSet = []
|
||||
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
|
||||
while (startDate.before(endDate)) {
|
||||
calendar.add(Calendar.DATE, 1)
|
||||
def nextDate = calendar.getTime()
|
||||
count++
|
||||
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
|
||||
def result = JSONObject.fromObject(doc.res_map.result)
|
||||
if(!result.buildYear || !result.avgPrice){
|
||||
return
|
||||
}
|
||||
def params = JSONObject.fromObject(doc.parameter_map)
|
||||
def key = "${params.address}_${params.size}".toString()
|
||||
if (!hashSet.contains(key)) {
|
||||
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
|
||||
data << item
|
||||
hashSet << key
|
||||
} else {
|
||||
println("已存在")
|
||||
}
|
||||
}
|
||||
startDate = nextDate
|
||||
}
|
||||
ExcelUtils.write("D:\\四川锦城${startDate.format('yyyyMM')}.xlsx", data, "查询日志")
|
||||
@@ -1,11 +1,17 @@
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
//import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
@@ -132,4 +138,70 @@ import org.apache.commons.lang3.StringUtils
|
||||
// item["E"] = RegionData.formatRegion(item["B"], item["C"])
|
||||
// println(item)
|
||||
//})
|
||||
CommonUtils.reFitAverage("常州","新北","龙城花园", "普通住宅")
|
||||
//def border = [13511506.7,3639168.61,13511516.8,3639404.86,13511515.27,3639416.92,13511738.21,3639384.94,13512007.72,3639336.35,13511984.92,3639074.96,13511722.75,3639121.08,13511506.7,3639168.61]
|
||||
//def coords = []
|
||||
//for(int i=0; i < border.size() - 1; i=i+2){
|
||||
// coords << "${border[i]},${border[i+1]}"
|
||||
//}
|
||||
//def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
//println(json)
|
||||
//def points = []
|
||||
//for(JSONObject it : json.result){
|
||||
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
// points << "${geo.lng},${geo.lat}"
|
||||
// println(" new AMap.LngLat(${geo.lng},${geo.lat}),")
|
||||
//}
|
||||
|
||||
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
|
||||
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
|
||||
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
|
||||
def coords = []
|
||||
def baidu_points = []
|
||||
def gaode_points = []
|
||||
for(int i=0; i < border.size() - 1; i=i+2){
|
||||
coords << "${border[i]},${border[i+1]}"
|
||||
if(coords.size() > 50){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
coords.clear()
|
||||
}
|
||||
}
|
||||
if(coords.size() > 0){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
}
|
||||
dbo.format_city = RegionData.formatCity(dbo.city)
|
||||
dbo.baidu_border = baidu_points.join(";")
|
||||
dbo.gaode_border = gaode_points.join(";")
|
||||
BAIDUMAP_BORDER_COLL.save(dbo)
|
||||
println(dbo)
|
||||
}
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
|
||||
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
|
||||
// if(seaweed){
|
||||
// item["A"] = seaweed._id.toString()
|
||||
// def names = [item['D']]
|
||||
// if(seaweed.alias){
|
||||
// names.addAll(seaweed.alias)
|
||||
// }
|
||||
// names.unique()
|
||||
// if(seaweed.lng){
|
||||
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
|
||||
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
|
||||
// }
|
||||
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
|
||||
// if(one){
|
||||
// item["I"] = one.baidu_border
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
@@ -669,7 +669,6 @@ def findRentInfoByRoom(String city, String region){
|
||||
|
||||
|
||||
def findRent(def city, def region, def name){
|
||||
DBCollection DISTRICT_STAT_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
|
||||
def query = new BasicDBObject()
|
||||
query.city = city
|
||||
query.region = region
|
||||
|
||||
+67
-47
@@ -14,6 +14,7 @@ import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.client.MongoCollection
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang3.StringUtils
|
||||
import org.dom4j.Document
|
||||
@@ -204,52 +205,71 @@ import org.dom4j.io.SAXReader
|
||||
//// }
|
||||
//// }
|
||||
//})
|
||||
DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
|
||||
statMap = [:]
|
||||
def s = new HashSet()
|
||||
int count = 0
|
||||
def data = [["城市", "地址"]]
|
||||
def query = new BasicDBObject("user_id", "1007751").append("c_date", new BasicDBObject('$gte', Date.parse("yyyyMMdd", "20210119")))
|
||||
println JSONObject.fromObject(query)
|
||||
println COMMAND_COLL.count(query)
|
||||
COMMAND_COLL.find(query).batchSize(10000).each { BasicDBObject log ->
|
||||
count++
|
||||
// def city = log.parameter_map.city
|
||||
// def stat = statMap.get(city, [:])
|
||||
// stat["调用次数"] = stat.get("调用次数", 0) + 1
|
||||
if (log.res_map && log.res_map.result && log.res_map.result.totalPrice) {
|
||||
// stat["估价成功次数"] = stat.get("估价成功次数", 0) + 1
|
||||
// if(log.res_map.result.calcType == 1){
|
||||
// stat["有样本"] = stat.get("有样本", 0) + 1
|
||||
// }else{
|
||||
// stat["无样本"] = stat.get("无样本", 0) + 1
|
||||
//DBCollection COMMAND_COLL = MongoUtils.getCollectionByName("business", "open_uri_log", "192.168.0.206", 27020)
|
||||
//statMap = [:]
|
||||
//def s = new HashSet()
|
||||
//int count = 0
|
||||
//def data = [["城市", "地址"]]
|
||||
//def query = new BasicDBObject("user_id", "1007751").append("c_date", new BasicDBObject('$gte', Date.parse("yyyyMMdd", "20210119")))
|
||||
//println JSONObject.fromObject(query)
|
||||
//println COMMAND_COLL.count(query)
|
||||
//COMMAND_COLL.find(query).batchSize(10000).each { BasicDBObject log ->
|
||||
// count++
|
||||
//// def city = log.parameter_map.city
|
||||
//// def stat = statMap.get(city, [:])
|
||||
//// stat["调用次数"] = stat.get("调用次数", 0) + 1
|
||||
// if (log.res_map && log.res_map.result && log.res_map.result.totalPrice) {
|
||||
//// stat["估价成功次数"] = stat.get("估价成功次数", 0) + 1
|
||||
//// if(log.res_map.result.calcType == 1){
|
||||
//// stat["有样本"] = stat.get("有样本", 0) + 1
|
||||
//// }else{
|
||||
//// stat["无样本"] = stat.get("无样本", 0) + 1
|
||||
//// }
|
||||
//// stat["总价"] = stat.get("总价", 0l) + log.res_map.result.totalPrice
|
||||
//// stat["总面积"] = stat.get("总面积", 0d) + log.parameter_map.size.toDouble()
|
||||
//
|
||||
// } else {
|
||||
// if (log.res_map.msg.toString().contains("该小区不存在")) {
|
||||
// println( [log.parameter_map.city, log.parameter_map.address])
|
||||
// data << [log.parameter_map.city, log.parameter_map.address]
|
||||
// }
|
||||
// stat["总价"] = stat.get("总价", 0l) + log.res_map.result.totalPrice
|
||||
// stat["总面积"] = stat.get("总面积", 0d) + log.parameter_map.size.toDouble()
|
||||
|
||||
} else {
|
||||
if (log.res_map.msg.toString().contains("该小区不存在")) {
|
||||
println( [log.parameter_map.city, log.parameter_map.address])
|
||||
data << [log.parameter_map.city, log.parameter_map.address]
|
||||
}
|
||||
// stat["估价失败次数"] = stat.get("估价失败次数", 0) + 1
|
||||
// stat["失败原因:" + log.res_map.msg] = stat.get("失败原因:" + log.res_map.msg, 0) + 1
|
||||
// s << "失败原因:" + log.res_map.msg
|
||||
}
|
||||
if (count % 10000 == 0) {
|
||||
println(count)
|
||||
}
|
||||
}
|
||||
data = data.unique()
|
||||
ExcelUtils.write("D:\\未匹配地址.xlsx", data, "结果")
|
||||
//key = ["调用次数", "估价成功次数", "有样本", "无样本", "总价", "总面积", "估价失败次数"]
|
||||
//key.addAll(s)
|
||||
//print(key.join("\t"))
|
||||
//[ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ].each {def city->
|
||||
// stat = statMap.get(city)
|
||||
// con = [city]
|
||||
// for( def k : key){
|
||||
// con << stat[k]
|
||||
//// stat["估价失败次数"] = stat.get("估价失败次数", 0) + 1
|
||||
//// stat["失败原因:" + log.res_map.msg] = stat.get("失败原因:" + log.res_map.msg, 0) + 1
|
||||
//// s << "失败原因:" + log.res_map.msg
|
||||
// }
|
||||
// println con.join("\t")
|
||||
//}
|
||||
// if (count % 10000 == 0) {
|
||||
// println(count)
|
||||
// }
|
||||
//}
|
||||
//data = data.unique()
|
||||
//ExcelUtils.write("D:\\未匹配地址.xlsx", data, "结果")
|
||||
////key = ["调用次数", "估价成功次数", "有样本", "无样本", "总价", "总面积", "估价失败次数"]
|
||||
////key.addAll(s)
|
||||
////print(key.join("\t"))
|
||||
////[ "苏州", "广州", "杭州", "大连", "重庆", "成都", "深圳", "厦门", "上海", "北京" ].each {def city->
|
||||
//// stat = statMap.get(city)
|
||||
//// con = [city]
|
||||
//// for( def k : key){
|
||||
//// con << stat[k]
|
||||
//// }
|
||||
//// println con.join("\t")
|
||||
////}
|
||||
//
|
||||
//DBCollection OPEN_URI_LOG_42 = MongoUtils.getCollectionByName("business", "open_uri_log", "log.mongo.db.fangjia.com", 52002)
|
||||
//def data = [["城市", "地址", "省", "市", "区", "开发区","乡镇", "路", "路号", "小区名", "楼栋号", "单元号", "室号", "标准化地址"]]
|
||||
//OPEN_URI_LOG_42.find(new BasicDBObject("user_id", "1007756")).each {BasicDBObject dbo->
|
||||
// data << [dbo.parameter_map.city, dbo.parameter_map.address, dbo.res_map.result.province, dbo.res_map.result.city, dbo.res_map.result.district, dbo.res_map.result.developmentZone, dbo.res_map.result.township, dbo.res_map.result.street,dbo.res_map.result.streetNumber,dbo.res_map.result.name,dbo.res_map.result.buildingNumber,dbo.res_map.result.unitNumber,dbo.res_map.result.roomNumber,dbo.res_map.result.formattedAddress]
|
||||
//}
|
||||
//ExcelUtils.write("D:\\建行测试地址.xlsx", data, "结果")
|
||||
DBCollection SQMETRE_RENT_COLL = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com")
|
||||
def query = new BasicDBObject()
|
||||
query.cat = "region"
|
||||
query.city= "上海"
|
||||
query.weekend = Date.parse("yyyyMMdd", "20210228")
|
||||
def doc = []
|
||||
def prices = []
|
||||
SQMETRE_RENT_COLL.find(query, new BasicDBObject("region", 1).append("avg_price", 1)).each {
|
||||
doc.add(new BasicDBObject("name", it.region+"区").append("value", it.avg_price))
|
||||
prices << it.avg_price
|
||||
}
|
||||
println(JSONArray.fromObject(doc).toString())
|
||||
+2
-13
@@ -1,19 +1,8 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
//DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\下载记录20200818-结果 - 副本.xlsx",{List item->
|
||||
|
||||
+3
-17
@@ -1,23 +1,9 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行学校数据\\上海学校_0811.xls",{List item->
|
||||
|
||||
@@ -5,13 +5,12 @@ import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
|
||||
//def data = []
|
||||
//data << ["城市", "区域", "小区名", "近一月", "近三月", "近6月", "近一年", "户数"]
|
||||
//DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("上海")).each {BasicDBObject district->
|
||||
@@ -36,7 +35,7 @@ import java.sql.Statement
|
||||
//ExcelUtils.write("D:\\上海成交统计.xlsx", data, "成交数据")
|
||||
|
||||
def find_rent_house_prices(String city, String region, String name){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -80,7 +79,7 @@ def find_rent_house_prices(String city, String region, String name){
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\上海成交统计.xlsx", {List item->
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-3
@@ -1,13 +1,12 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -16,7 +15,7 @@ import java.sql.Statement
|
||||
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
|
||||
|
||||
def String hasElevator(BasicDBObject seaweed){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+12
-34
@@ -1,40 +1,18 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
//ElasticUtils.executeSqlQuery("select * from seawater where p_date > 20210407 limit 10",{ Map row->
|
||||
// println(row.name)
|
||||
//}, "192.168.0.216")
|
||||
|
||||
ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\新增小区.xlsx", 0, 0).each {List item->
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.city = 1
|
||||
keys.region = 1
|
||||
keys.name = 1
|
||||
keys.estate_type2 = 1
|
||||
def query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
|
||||
query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
|
||||
DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
|
||||
if(!DbUtils.findDistrictStat(dbo, new Date())){
|
||||
CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
|
||||
println(dbo)
|
||||
}
|
||||
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
|
||||
// println(dbo)
|
||||
}
|
||||
}
|
||||
//["佛山","济南","北京","广州","深圳"].each {def city->
|
||||
// def keys = new BasicDBObject()
|
||||
// keys.city = 1
|
||||
// keys.region = 1
|
||||
// keys.name = 1
|
||||
// keys.estate_type2 = 1
|
||||
// def query = DbUtils.buildSeaweedQuery(city)
|
||||
// query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
|
||||
// DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
|
||||
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
|
||||
// println(dbo)
|
||||
// }
|
||||
//ElasticUtils.executeSimpleQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "ÐÂãþÈý´å")).each {
|
||||
// println(it)
|
||||
//}
|
||||
|
||||
ElasticUtils.executeQuery("192.168.0.216", "seawater-2021", QueryBuilders.termQuery("district_name", "ÐÂãþÈý´å"),{
|
||||
println(it)
|
||||
})
|
||||
|
||||
|
||||
+3
-9
@@ -1,20 +1,14 @@
|
||||
package com.fangjia.script.yinjihaun.price
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
|
||||
Reference in New Issue
Block a user