升级Es驱动版本
This commit is contained in:
@@ -277,6 +277,39 @@ public class BaiDuApiUtils {
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
*
|
||||
* @param coords
|
||||
* @param from
|
||||
* @return
|
||||
*/
|
||||
public static JSONObject geoconv(String coords,int from){
|
||||
Map<String,String> params = new HashMap<String, String>();
|
||||
params.put("output", "json");
|
||||
params.put("ak", AK);
|
||||
params.put("coords", coords);
|
||||
params.put("from", String.valueOf(from));
|
||||
HttpRequest request = new HttpRequest();
|
||||
try{
|
||||
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
JSONObject json = JSONObject.fromObject(result);
|
||||
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
|
||||
changeAK();
|
||||
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
|
||||
json = JSONObject.fromObject(result);
|
||||
Thread.sleep(1000 * 60);
|
||||
}
|
||||
if(json.getInt("status") >= 300){
|
||||
changeAK();
|
||||
json = geoconv(coords, from);
|
||||
}
|
||||
return json.getJSONArray("result").getJSONObject(0);
|
||||
}catch (Exception e){
|
||||
e.printStackTrace();
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
public static JSONObject geoconv(String coords, int from, int to){
|
||||
Map<String,String> params = new HashMap<String, String>();
|
||||
|
||||
@@ -17,8 +17,8 @@
|
||||
<artifactId>maven-compiler-plugin</artifactId>
|
||||
<version>2.3.2</version>
|
||||
<configuration>
|
||||
<source>1.7</source>
|
||||
<target>1.7</target>
|
||||
<source>1.8</source>
|
||||
<target>1.8</target>
|
||||
<encoding>GBK</encoding>
|
||||
</configuration>
|
||||
</plugin>
|
||||
@@ -179,7 +179,7 @@
|
||||
<dependency>
|
||||
<groupId>org.apache.httpcomponents</groupId>
|
||||
<artifactId>httpclient</artifactId>
|
||||
<version>4.4.1</version>
|
||||
<version>4.5.10</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.ant</groupId>
|
||||
@@ -315,16 +315,6 @@
|
||||
<artifactId>metadata-extractor</artifactId>
|
||||
<version>2.4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.http</groupId>
|
||||
<artifactId>httpasyncclient</artifactId>
|
||||
<version>4.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.http</groupId>
|
||||
<artifactId>httpcore-nio</artifactId>
|
||||
<version>4.1.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.sanselan</groupId>
|
||||
<artifactId>sanselan</artifactId>
|
||||
@@ -448,23 +438,18 @@
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>x-pack-transport</artifactId>
|
||||
<version>6.4.2</version>
|
||||
<artifactId>elasticsearch-rest-high-level-client</artifactId>
|
||||
<version>7.11.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.plugin</groupId>
|
||||
<artifactId>x-pack-sql-jdbc</artifactId>
|
||||
<version>6.4.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.elasticsearch.client</groupId>
|
||||
<artifactId>transport</artifactId>
|
||||
<version>6.4.2</version>
|
||||
<version>7.11.2</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.lucene</groupId>
|
||||
<artifactId>lucene-core</artifactId>
|
||||
<version>7.4.0</version>
|
||||
<version>8.7.0</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.commons</groupId>
|
||||
|
||||
+2
-2
@@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
|
||||
|
||||
def findHouseMaxArea(String city, String region, String name, List alias) {
|
||||
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24)
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-3
@@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection
|
||||
import com.mongodb.client.model.UpdateOptions
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it->
|
||||
names << it.name
|
||||
}
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
|
||||
|
||||
def findHouseCount(String city, String region, String name, List alias, int months) {
|
||||
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months)
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+5
-9
@@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement;
|
||||
import java.sql.Statement
|
||||
|
||||
/**
|
||||
*
|
||||
* 历史均价计算
|
||||
@@ -27,7 +23,7 @@ import java.sql.Statement;
|
||||
*/
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class HistoricalAverageRevaluationTask extends BaseBatch{
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
|
||||
public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat')
|
||||
|
||||
+2
-13
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.avgprice
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
@@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.Bytes
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement;
|
||||
import com.mongodb.*
|
||||
|
||||
/**
|
||||
* 无样本小区历史遗漏数据补充
|
||||
|
||||
+2
-2
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
@@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{
|
||||
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -4,13 +4,13 @@ import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class AddEsHouseField {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+2
-3
@@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
@@ -63,7 +62,7 @@ class AddThirdPriceDistrict {
|
||||
def province = null
|
||||
def resAddrMap = [:]
|
||||
def filterEstateTypeMap = [:]
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.esms.common.util.CommonUtil
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.huaban.analysis.jieba.JiebaSegmenter
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -485,7 +485,7 @@ class CrawlDataMatch {
|
||||
}
|
||||
|
||||
def findAllAddress(def cityName){
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -1,24 +1,19 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.SQLException
|
||||
import java.sql.Statement
|
||||
|
||||
class EsHouseStat {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class EsSearchUtils {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
@@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
@@ -20,7 +19,7 @@ class FZDataExport {
|
||||
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
|
||||
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+2
-3
@@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{
|
||||
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -3,14 +3,14 @@ package com.fangjia.script.gj
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
class RoomStat {
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
@@ -1,30 +1,25 @@
|
||||
package com.fangjia.script.gj
|
||||
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
import java.util.zip.GZIPInputStream
|
||||
|
||||
import static org.elasticsearch.index.query.QueryBuilders.idsQuery
|
||||
|
||||
class TradeDataMatch {
|
||||
|
||||
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query"
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
|
||||
def final AtomicInteger count_ = new AtomicInteger(0)
|
||||
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-4
File diff suppressed because one or more lines are too long
+4
-12
@@ -1,24 +1,17 @@
|
||||
package com.fangjia.script.gj.ccb
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.index.query.BoolQueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.SimpleDateFormat
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
@@ -26,12 +19,11 @@ import java.util.concurrent.Executors
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
|
||||
class HouseExport {
|
||||
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+2
-4
@@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
class ExportData {
|
||||
|
||||
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+3
-8
@@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.gj.EsSearchUtils
|
||||
import com.fangjia.script.gj.district.FangjiaClientUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.GeoHelper
|
||||
import com.fangjia.script.util.GeoUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -27,7 +22,7 @@ class DistrictWash {
|
||||
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
|
||||
static def totalCount = new AtomicLong()
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+17
-27
@@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.DecimalFormat
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import net.sf.json.JSONObject
|
||||
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils;
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
import com.fangjia.script.util.ExcelParser
|
||||
import com.fangjia.script.common.DataHandleUtils;
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.common.DataHandleUtils
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
import org.bson.types.ObjectId
|
||||
|
||||
import com.fangjia.script.util.MongoUtils;
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
|
||||
import java.util.Map.Entry;
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.codec.digest.DigestUtils
|
||||
import org.apache.commons.io.FileUtils
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.poi.hssf.usermodel.HSSFWorkbook
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.DecimalFormat
|
||||
import java.util.Map.Entry
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
class CZBlankDataExport {
|
||||
|
||||
@@ -62,7 +52,7 @@ class CZBlankDataExport {
|
||||
def directionMap = [1:"ÄÏ",2:"±±",3:"Äϱ±",4:"¶«",5:"Î÷",6:"¶«Î÷",7:"¶«ÄÏ",8:"Î÷ÄÏ",9:"¶«±±",10:"Î÷±±"]
|
||||
|
||||
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
-1
@@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
/**
|
||||
* 江南银行线上数据交付
|
||||
|
||||
+2
-17
@@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.crawl.CrawlCommandConsumerBatch
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ElasticUtils
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBList
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.BatchUpdateException
|
||||
import java.sql.ResultSet
|
||||
import java.text.NumberFormat
|
||||
import java.util.Map.Entry
|
||||
import java.util.concurrent.BlockingQueue
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.LinkedBlockingDeque
|
||||
import java.util.concurrent.LinkedBlockingQueue
|
||||
import java.util.concurrent.ThreadPoolExecutor
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.regex.Pattern
|
||||
|
||||
@BatchOwner(Watcher.WeiLei)
|
||||
class HousingOverviewExportBatch extends BaseBatch {
|
||||
|
||||
static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>())
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+4
-7
@@ -1,6 +1,5 @@
|
||||
package com.fangjia.script.jusfoun
|
||||
|
||||
|
||||
import com.alibaba.fastjson.JSONObject
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
@@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.index.query.QueryBuilder
|
||||
import org.elasticsearch.index.query.QueryBuilders
|
||||
import org.elasticsearch.search.SearchHit
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import redis.clients.jedis.JedisPool
|
||||
import redis.clients.jedis.JedisPoolConfig
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.*
|
||||
import java.text.DateFormat
|
||||
@@ -56,7 +53,7 @@ class JusfounCommon {
|
||||
public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
|
||||
public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
|
||||
|
||||
private static JdbcDataSource dataSource = null
|
||||
private static EsDataSource dataSource = null
|
||||
|
||||
private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>();
|
||||
private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>();
|
||||
@@ -390,7 +387,7 @@ class JusfounCommon {
|
||||
return DataUtil.hanyuQuanpin(hanzi, '')
|
||||
}
|
||||
|
||||
static JdbcDataSource getESDataSource() {
|
||||
static EsDataSource getESDataSource() {
|
||||
if (dataSource != null) {
|
||||
return dataSource;
|
||||
} else {
|
||||
@@ -399,7 +396,7 @@ class JusfounCommon {
|
||||
return dataSource;
|
||||
}
|
||||
try {
|
||||
dataSource = new JdbcDataSource();
|
||||
dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties();
|
||||
|
||||
+2
-2
@@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
|
||||
import org.apache.commons.math3.stat.descriptive.moment.Mean
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
def List find_rent_house_prices(String city, String region, String name, Date start, Date end){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+5
-8
@@ -1,20 +1,17 @@
|
||||
package com.fangjia.script.log
|
||||
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class MongoDBLogDisMonitorBatch extends BaseBatch {
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
|
||||
+6
-8
@@ -1,20 +1,18 @@
|
||||
package com.fangjia.script.log
|
||||
|
||||
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
@BatchOwner(Watcher.JiangYaHui)
|
||||
class MongoDBLogMonitorBatch extends BaseBatch {
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
|
||||
|
||||
|
||||
+3
-2
@@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.atomic.AtomicInteger
|
||||
|
||||
@@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch {
|
||||
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final JdbcDataSource dataSource = new JdbcDataSource()
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+6
-4
@@ -53,8 +53,10 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
void process(Object args) {
|
||||
|
||||
Map userInfo = getUserInfo()
|
||||
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
|
||||
def endDate = DateUtil.truncateDay(new Date())
|
||||
// def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
|
||||
// def endDate = DateUtil.truncateDay(new Date())
|
||||
def startDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210308"))
|
||||
def endDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210314"))
|
||||
|
||||
BasicDBObject query = new BasicDBObject()
|
||||
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
|
||||
@@ -64,7 +66,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
long count = 0
|
||||
long total = cursor.count()
|
||||
print("total="+total)
|
||||
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia")
|
||||
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("seawater.es.db.fangjia.com")
|
||||
for(BasicDBObject log : cursor){
|
||||
Map<String,Object> parameter_map = (Map) log.get("parameter_map")
|
||||
if(parameter_map == null) {
|
||||
@@ -128,7 +130,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
|
||||
cmd.put("user_name", user.user_name)
|
||||
cmd.put("company_name", user.company)
|
||||
}
|
||||
bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd))
|
||||
bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd))
|
||||
println(cmd)
|
||||
count++
|
||||
if(count % 100 == 0){
|
||||
|
||||
+11
-37
@@ -1,50 +1,24 @@
|
||||
package com.fangjia.script.trade
|
||||
///µ¼Èë¼Û¸ñÖ¸Êý
|
||||
import com.fangjia.script.BaseBatch;
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.script.BaseBatch
|
||||
|
||||
///导入价格指数
|
||||
|
||||
import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
|
||||
import org.apache.commons.io.FileUtils;
|
||||
|
||||
|
||||
|
||||
import org.apache.commons.lang.StringUtils;
|
||||
import org.apache.poi.xssf.usermodel.XSSFCell
|
||||
import org.apache.poi.xssf.usermodel.XSSFWorkbook
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.QueryOperators
|
||||
import com.fangjia.script.util.DateUtils;
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection;
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.Mongo
|
||||
|
||||
import groovy.json.JsonSlurper
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.lang.math.RandomUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.text.NumberFormat;
|
||||
import java.util.regex.Matcher
|
||||
import java.util.regex.Pattern
|
||||
|
||||
import com.fangjia.core.util.DBIterator
|
||||
|
||||
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.StringUtil;
|
||||
import com.fangjia.datastore.driver.DSCollection;
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
import com.fangjia.script.commData.CommonDatautils;
|
||||
import com.ibm.icu.text.SimpleDateFormat;
|
||||
import com.fangjia.storehouse.DBIterator
|
||||
import com.fangjia.storehouse.DBIterator;
|
||||
import com.fangjia.storehouse.StorehouseDB;
|
||||
|
||||
import org.apache.commons.lang.math.RandomUtils;
|
||||
@BatchOwner(Watcher.GaoJuan)
|
||||
class ImportIndexStats extends BaseBatch{
|
||||
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
|
||||
@@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{
|
||||
public static final def dingjiYear = 2016;
|
||||
|
||||
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
private static final JdbcDataSource dataSource = new JdbcDataSource()
|
||||
private static final EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-8
@@ -1,17 +1,12 @@
|
||||
package com.fangjia.script.trade
|
||||
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtil
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import com.mongodb.QueryOperators
|
||||
import com.mongodb.*
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
@@ -22,7 +17,7 @@ class TradeDealPriceStat {
|
||||
|
||||
|
||||
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -3,7 +3,7 @@ package com.fangjia.script.usahouse
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
import java.util.concurrent.CountDownLatch
|
||||
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
|
||||
|
||||
class AddressMatch {
|
||||
|
||||
public static JdbcDataSource dataSource = new JdbcDataSource()
|
||||
public static EsDataSource dataSource = new EsDataSource()
|
||||
private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
static {
|
||||
dataSource.setUrl(address)
|
||||
|
||||
+1
-4
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.usahouse
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.httpclient.UserAgents
|
||||
import com.fangjia.script.BaseBatch
|
||||
@@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner
|
||||
import com.fangjia.script.Watcher
|
||||
import com.fangjia.script.crawl.JQueryUtils
|
||||
import com.fangjia.script.thirdPriceCrawl.CrawalUtil
|
||||
import com.fangjia.script.util.HttpClient
|
||||
import com.fangjia.script.util.JacksonUtil
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSON
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.jsoup.Jsoup
|
||||
import org.jsoup.nodes.Document
|
||||
import org.jsoup.select.Elements
|
||||
|
||||
+92
-37
@@ -1,69 +1,125 @@
|
||||
package com.fangjia.script.util
|
||||
|
||||
import org.apache.commons.lang3.SystemUtils
|
||||
|
||||
import org.apache.http.HttpHost
|
||||
import org.apache.http.auth.AuthScope
|
||||
import org.apache.http.auth.UsernamePasswordCredentials
|
||||
import org.apache.http.client.CredentialsProvider
|
||||
import org.apache.http.client.config.RequestConfig
|
||||
import org.apache.http.impl.client.BasicCredentialsProvider
|
||||
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
|
||||
import org.elasticsearch.action.ActionListener
|
||||
import org.elasticsearch.action.bulk.BackoffPolicy
|
||||
import org.elasticsearch.action.bulk.BulkProcessor
|
||||
import org.elasticsearch.action.bulk.BulkRequest
|
||||
import org.elasticsearch.action.bulk.BulkResponse
|
||||
import org.elasticsearch.client.transport.TransportClient
|
||||
import org.elasticsearch.common.settings.Settings
|
||||
import org.elasticsearch.common.transport.TransportAddress
|
||||
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient
|
||||
import org.elasticsearch.client.RestClient
|
||||
import org.elasticsearch.client.RestClientBuilder
|
||||
import org.elasticsearch.client.RestHighLevelClient
|
||||
import org.elasticsearch.client.RequestOptions
|
||||
import org.elasticsearch.common.unit.TimeValue
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.function.BiConsumer
|
||||
|
||||
/**
|
||||
* 功能说明:Elasticsearch工具类
|
||||
*/
|
||||
class ElasticsearchHelper {
|
||||
class ElasticsearchHelper {
|
||||
|
||||
|
||||
static getClient(String host, String clusterName = "fangjia"){
|
||||
Settings.Builder builder = Settings.builder()
|
||||
.put("cluster.name", clusterName)
|
||||
.put("client.transport.sniff", false)
|
||||
.put("client.transport.ping_timeout", "60s")
|
||||
.put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3")
|
||||
.put("xpack.security.transport.ssl.enabled", "true")
|
||||
if(SystemUtils.IS_OS_WINDOWS){
|
||||
builder
|
||||
.put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
|
||||
.put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
|
||||
}else{
|
||||
builder
|
||||
.put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12")
|
||||
.put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12")
|
||||
static getEsDataSource(String host){
|
||||
EsDataSource dataSource = new EsDataSource ();
|
||||
dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
|
||||
Properties connectionProperties = new Properties();
|
||||
connectionProperties.setProperty("user","developer");
|
||||
connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
|
||||
dataSource.setProperties(connectionProperties);
|
||||
return dataSource;
|
||||
}
|
||||
|
||||
static executeQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
|
||||
EsDataSource dataSource = getEsDataSource(host)
|
||||
Connection connection = dataSource.getConnection()
|
||||
Statement statement = connection.createStatement()
|
||||
ResultSet rs = statement.executeQuery(sql)
|
||||
def columns = []
|
||||
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
|
||||
columns << rs.getMetaData().getColumnName(i)
|
||||
}
|
||||
builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build()
|
||||
Settings settings = builder.build()
|
||||
return new PreBuiltXPackTransportClient(settings)
|
||||
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300))
|
||||
rs.getMetaData().getColumnCount()
|
||||
while(rs.next()){
|
||||
def row = [:]
|
||||
for (String col : columns){
|
||||
row[col] = rs.getObject(col)
|
||||
}
|
||||
closure.call(row)
|
||||
}
|
||||
rs.close()
|
||||
connection.close()
|
||||
}
|
||||
|
||||
|
||||
static RestHighLevelClient getClient(String host) {
|
||||
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
|
||||
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
|
||||
@Override
|
||||
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
|
||||
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
|
||||
requestConfigBuilder.setSocketTimeout(1000 * 60);
|
||||
return requestConfigBuilder;
|
||||
}
|
||||
})
|
||||
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
|
||||
@Override
|
||||
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
|
||||
httpAsyncClientBuilder.disableAuthCaching();
|
||||
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
|
||||
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
|
||||
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
|
||||
return httpAsyncClientBuilder;
|
||||
}
|
||||
})
|
||||
return new RestHighLevelClient(builder)
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* 创建一个ES批处理器
|
||||
* @param clusterName
|
||||
* @param host
|
||||
* @return
|
||||
*/
|
||||
static createBulkProcessor( String host, String clusterName){
|
||||
TransportClient client = getClient(host, clusterName)
|
||||
BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {
|
||||
static createBulkProcessor(String host) {
|
||||
BulkProcessor.Listener listener = new BulkProcessor.Listener() {
|
||||
@Override
|
||||
void beforeBulk(long executionId, BulkRequest request) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request, BulkResponse response) {
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
BulkResponse response) {
|
||||
}
|
||||
|
||||
@Override
|
||||
void afterBulk(long executionId, BulkRequest request, Throwable failure) {
|
||||
failure.printStackTrace()
|
||||
void afterBulk(long executionId, BulkRequest request,
|
||||
Throwable failure) {
|
||||
}
|
||||
}).setBulkActions(10000).setConcurrentRequests(20).build()
|
||||
};
|
||||
RestHighLevelClient client = getClient(host);
|
||||
|
||||
return bulkProcessor
|
||||
return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
|
||||
@Override
|
||||
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
|
||||
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
|
||||
}
|
||||
}, listener).setBulkActions(10000)
|
||||
.setConcurrentRequests(20)
|
||||
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
|
||||
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -71,14 +127,13 @@ class ElasticsearchHelper {
|
||||
* @param
|
||||
* @return
|
||||
*/
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor){
|
||||
static closeBulkProcessor(BulkProcessor bulkProcessor) {
|
||||
|
||||
bulkProcessor.awaitClose(10, TimeUnit.MINUTES)
|
||||
bulkProcessor.awaitClose(10, TimeUnit.DAYS)
|
||||
bulkProcessor.flush()
|
||||
bulkProcessor.close()
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
+3
-3
@@ -1,6 +1,6 @@
|
||||
package com.fangjia.script.weilei
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.Bytes
|
||||
import com.mongodb.DBCollection
|
||||
@@ -8,7 +8,7 @@ import com.mongodb.DBCursor
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -16,7 +16,7 @@ import java.sql.SQLException
|
||||
import java.sql.Statement
|
||||
import java.text.SimpleDateFormat
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties();
|
||||
|
||||
@@ -1,31 +1,16 @@
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.Jms
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.CommandResult
|
||||
import com.mongodb.DB
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.*;
|
||||
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import static com.mongodb.client.model.Filters.and
|
||||
import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
|
||||
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
|
||||
@@ -43,7 +28,7 @@ cityList.each { def city ->
|
||||
}
|
||||
})
|
||||
roots.each { def root ->
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-14
@@ -1,23 +1,12 @@
|
||||
package com.fangjia.script.wenyan.address
|
||||
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.yinjihaun.util.QQKeyPool
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.lucene.index.Term
|
||||
import org.apache.lucene.search.BooleanClause
|
||||
import org.apache.lucene.search.BooleanQuery
|
||||
import org.apache.lucene.search.TermQuery
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -44,7 +33,7 @@ import java.sql.Statement
|
||||
// data << (k.split("_") as List)
|
||||
//}
|
||||
//ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û")
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-3
@@ -1,20 +1,20 @@
|
||||
package com.fangjia.script.wenyan.avgPrice
|
||||
|
||||
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.apache.solr.client.solrj.util.ClientUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
def findEstateType(String city, String region, String name, List alias) {
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-7
@@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils
|
||||
import org.apache.commons.math3.ml.clustering.CentroidCluster
|
||||
import org.apache.commons.math3.ml.clustering.DoublePoint
|
||||
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017)
|
||||
|
||||
def findHouse(BasicDBObject district){
|
||||
@@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){
|
||||
sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ")
|
||||
sqlBuffer.append(" ORDER BY p_date desc")
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s";
|
||||
dataSource.setUrl(address);
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
package com.fangjia.script.wenyan.seawater
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -1,13 +1,13 @@
|
||||
package com.fangjia.script.wenyan.seawater
|
||||
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
JdbcDataSource dataSource = new JdbcDataSource();
|
||||
EsDataSource dataSource = new EsDataSource();
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-14
@@ -1,20 +1,12 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.and
|
||||
import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
//
|
||||
//def data = [["城市", "区域", "小区名", "2018", "全部"]]
|
||||
//cityList = ["上海"]
|
||||
@@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq
|
||||
|
||||
//HttpRequest httpRequest = new HttpRequest()
|
||||
//println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8")
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config")
|
||||
AVERAGER_CONFIG.find().each {Document doc->
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20201101", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20201201", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
+3
-9
@@ -1,19 +1,13 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.Mongo
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -61,7 +55,7 @@ import java.sql.Statement
|
||||
//println(bigMarkupCount)
|
||||
|
||||
def countHouses(String city, String region, String name){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+3
-15
@@ -1,28 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.Constants
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCollection
|
||||
import com.fangjia.datastore.driver.DataStore
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.bson.Document
|
||||
import com.mongodb.client.MongoCollection
|
||||
import org.bson.types.ObjectId
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import java.util.function.Consumer
|
||||
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
import com.fangjia.core.util.Mongo3Utils
|
||||
def poiData = Collections.synchronizedList([])
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item ->
|
||||
@@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([])
|
||||
// println(item)
|
||||
//},1)
|
||||
def findHouseCount(String city, String region, String name, List alias) {
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-2
@@ -11,8 +11,8 @@ import org.bson.Document
|
||||
import static com.mongodb.client.model.Filters.*
|
||||
|
||||
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
|
||||
def startDate = DateUtil.parseDate("20210201", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
|
||||
def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
|
||||
Calendar calendar = Calendar.getInstance()
|
||||
calendar.setTime(startDate)
|
||||
def count = 0
|
||||
|
||||
@@ -1,11 +1,17 @@
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.core.util.BaiDuApiUtils
|
||||
import com.fangjia.core.util.DataUtil
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.core.util.GeoUtils
|
||||
import com.fangjia.core.util.GeocodeUtils
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.datastore.driver.DSCursor
|
||||
import com.fangjia.datastore.driver.FacetResult
|
||||
import com.fangjia.script.util.BaiduMapUtils
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
//import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
@@ -132,4 +138,70 @@ import org.apache.commons.lang3.StringUtils
|
||||
// item["E"] = RegionData.formatRegion(item["B"], item["C"])
|
||||
// println(item)
|
||||
//})
|
||||
CommonUtils.reFitAverage("常州","新北","龙城花园", "普通住宅")
|
||||
//def border = [13511506.7,3639168.61,13511516.8,3639404.86,13511515.27,3639416.92,13511738.21,3639384.94,13512007.72,3639336.35,13511984.92,3639074.96,13511722.75,3639121.08,13511506.7,3639168.61]
|
||||
//def coords = []
|
||||
//for(int i=0; i < border.size() - 1; i=i+2){
|
||||
// coords << "${border[i]},${border[i+1]}"
|
||||
//}
|
||||
//def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
//println(json)
|
||||
//def points = []
|
||||
//for(JSONObject it : json.result){
|
||||
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
// points << "${geo.lng},${geo.lat}"
|
||||
// println(" new AMap.LngLat(${geo.lng},${geo.lat}),")
|
||||
//}
|
||||
|
||||
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
|
||||
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
|
||||
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
|
||||
def coords = []
|
||||
def baidu_points = []
|
||||
def gaode_points = []
|
||||
for(int i=0; i < border.size() - 1; i=i+2){
|
||||
coords << "${border[i]},${border[i+1]}"
|
||||
if(coords.size() > 50){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
coords.clear()
|
||||
}
|
||||
}
|
||||
if(coords.size() > 0){
|
||||
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
|
||||
for(JSONObject it : json.result){
|
||||
baidu_points << "${it.x},${it.y}"
|
||||
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
|
||||
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
|
||||
}
|
||||
}
|
||||
dbo.format_city = RegionData.formatCity(dbo.city)
|
||||
dbo.baidu_border = baidu_points.join(";")
|
||||
dbo.gaode_border = gaode_points.join(";")
|
||||
BAIDUMAP_BORDER_COLL.save(dbo)
|
||||
println(dbo)
|
||||
}
|
||||
|
||||
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
|
||||
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
|
||||
// if(seaweed){
|
||||
// item["A"] = seaweed._id.toString()
|
||||
// def names = [item['D']]
|
||||
// if(seaweed.alias){
|
||||
// names.addAll(seaweed.alias)
|
||||
// }
|
||||
// names.unique()
|
||||
// if(seaweed.lng){
|
||||
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
|
||||
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
|
||||
// }
|
||||
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
|
||||
// if(one){
|
||||
// item["I"] = one.baidu_border
|
||||
// }
|
||||
// println(item)
|
||||
// }
|
||||
//})
|
||||
@@ -669,7 +669,6 @@ def findRentInfoByRoom(String city, String region){
|
||||
|
||||
|
||||
def findRent(def city, def region, def name){
|
||||
DBCollection DISTRICT_STAT_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
|
||||
def query = new BasicDBObject()
|
||||
query.city = city
|
||||
query.region = region
|
||||
|
||||
+2
-13
@@ -1,19 +1,8 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONArray
|
||||
import net.sf.json.JSONObject
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
//DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
|
||||
////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\下载记录20200818-结果 - 副本.xlsx",{List item->
|
||||
|
||||
+3
-17
@@ -1,23 +1,9 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DataUtil
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.FangJiaAreaMapUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.fangjia.script.util.ProvinceData
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.mongodb.DBCollection
|
||||
|
||||
//
|
||||
//ExcelUtils.multiThreadingModify("D:\\数据\\建行学校数据\\上海学校_0811.xls",{List item->
|
||||
|
||||
@@ -5,13 +5,12 @@ import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
import java.sql.Statement
|
||||
|
||||
|
||||
//def data = []
|
||||
//data << ["城市", "区域", "小区名", "近一月", "近三月", "近6月", "近一年", "户数"]
|
||||
//DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("上海")).each {BasicDBObject district->
|
||||
@@ -36,7 +35,7 @@ import java.sql.Statement
|
||||
//ExcelUtils.write("D:\\上海成交统计.xlsx", data, "成交数据")
|
||||
|
||||
def find_rent_house_prices(String city, String region, String name){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
@@ -80,7 +79,7 @@ def find_rent_house_prices(String city, String region, String name){
|
||||
}
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\上海成交统计.xlsx", {List item->
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+2
-3
@@ -1,13 +1,12 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
|
||||
import com.fangjia.script.util.*
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import com.mongodb.DBObject
|
||||
import net.sf.json.JSONObject
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.Connection
|
||||
import java.sql.ResultSet
|
||||
@@ -16,7 +15,7 @@ import java.sql.Statement
|
||||
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
|
||||
|
||||
def String hasElevator(BasicDBObject seaweed){
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
|
||||
+4
-37
@@ -1,40 +1,7 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.script.util.ElasticsearchHelper
|
||||
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\新增小区.xlsx", 0, 0).each {List item->
|
||||
|
||||
def keys = new BasicDBObject()
|
||||
keys.city = 1
|
||||
keys.region = 1
|
||||
keys.name = 1
|
||||
keys.estate_type2 = 1
|
||||
def query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
|
||||
query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
|
||||
DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
|
||||
if(!DbUtils.findDistrictStat(dbo, new Date())){
|
||||
CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
|
||||
println(dbo)
|
||||
}
|
||||
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
|
||||
// println(dbo)
|
||||
}
|
||||
}
|
||||
//["佛山","济南","北京","广州","深圳"].each {def city->
|
||||
// def keys = new BasicDBObject()
|
||||
// keys.city = 1
|
||||
// keys.region = 1
|
||||
// keys.name = 1
|
||||
// keys.estate_type2 = 1
|
||||
// def query = DbUtils.buildSeaweedQuery(city)
|
||||
// query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
|
||||
// DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
|
||||
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
|
||||
// println(dbo)
|
||||
// }
|
||||
//}
|
||||
ElasticsearchHelper.executeQuery("select * from seawater where p_date > 20210407 limit 10",{Map row->
|
||||
println(row.name)
|
||||
}, "192.168.0.216")
|
||||
+3
-9
@@ -1,20 +1,14 @@
|
||||
package com.fangjia.script.yinjihaun.price
|
||||
|
||||
import com.fangjia.core.util.HttpRequest
|
||||
import com.fangjia.script.commData.CommonDatautils
|
||||
import com.fangjia.script.util.DateUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
|
||||
import com.fangjia.script.util.MongoUtils
|
||||
import com.fangjia.script.util.OpenApiClient
|
||||
import com.mongodb.BasicDBObject
|
||||
import com.mongodb.DBCollection
|
||||
import org.apache.commons.lang.StringUtils
|
||||
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
|
||||
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
|
||||
|
||||
import java.sql.ResultSet
|
||||
|
||||
String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
|
||||
JdbcDataSource dataSource = new JdbcDataSource()
|
||||
EsDataSource dataSource = new EsDataSource()
|
||||
dataSource.setUrl(address)
|
||||
Properties connectionProperties = new Properties()
|
||||
connectionProperties.put("user", "developer")
|
||||
|
||||
Reference in New Issue
Block a user