升级Es驱动版本

This commit is contained in:
2021-04-08 13:32:52 +08:00
parent e4a18340bb
commit 32d1f77ec7
54 changed files with 361 additions and 453 deletions
@@ -278,6 +278,39 @@ public class BaiDuApiUtils {
return null; return null;
} }
/**
*
* @param coords
* @param from
* @return
*/
public static JSONObject geoconv(String coords,int from){
Map<String,String> params = new HashMap<String, String>();
params.put("output", "json");
params.put("ak", AK);
params.put("coords", coords);
params.put("from", String.valueOf(from));
HttpRequest request = new HttpRequest();
try{
String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
JSONObject json = JSONObject.fromObject(result);
while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){
changeAK();
result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8");
json = JSONObject.fromObject(result);
Thread.sleep(1000 * 60);
}
if(json.getInt("status") >= 300){
changeAK();
json = geoconv(coords, from);
}
return json.getJSONArray("result").getJSONObject(0);
}catch (Exception e){
e.printStackTrace();
}
return null;
}
public static JSONObject geoconv(String coords, int from, int to){ public static JSONObject geoconv(String coords, int from, int to){
Map<String,String> params = new HashMap<String, String>(); Map<String,String> params = new HashMap<String, String>();
params.put("output", "json"); params.put("output", "json");
+7 -22
View File
@@ -17,8 +17,8 @@
<artifactId>maven-compiler-plugin</artifactId> <artifactId>maven-compiler-plugin</artifactId>
<version>2.3.2</version> <version>2.3.2</version>
<configuration> <configuration>
<source>1.7</source> <source>1.8</source>
<target>1.7</target> <target>1.8</target>
<encoding>GBK</encoding> <encoding>GBK</encoding>
</configuration> </configuration>
</plugin> </plugin>
@@ -179,7 +179,7 @@
<dependency> <dependency>
<groupId>org.apache.httpcomponents</groupId> <groupId>org.apache.httpcomponents</groupId>
<artifactId>httpclient</artifactId> <artifactId>httpclient</artifactId>
<version>4.4.1</version> <version>4.5.10</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.ant</groupId> <groupId>org.apache.ant</groupId>
@@ -315,16 +315,6 @@
<artifactId>metadata-extractor</artifactId> <artifactId>metadata-extractor</artifactId>
<version>2.4.0</version> <version>2.4.0</version>
</dependency> </dependency>
<dependency>
<groupId>org.apache.http</groupId>
<artifactId>httpasyncclient</artifactId>
<version>4.0</version>
</dependency>
<dependency>
<groupId>org.apache.http</groupId>
<artifactId>httpcore-nio</artifactId>
<version>4.1.1</version>
</dependency>
<dependency> <dependency>
<groupId>org.apache.sanselan</groupId> <groupId>org.apache.sanselan</groupId>
<artifactId>sanselan</artifactId> <artifactId>sanselan</artifactId>
@@ -448,23 +438,18 @@
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.elasticsearch.client</groupId> <groupId>org.elasticsearch.client</groupId>
<artifactId>x-pack-transport</artifactId> <artifactId>elasticsearch-rest-high-level-client</artifactId>
<version>6.4.2</version> <version>7.11.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.elasticsearch.plugin</groupId> <groupId>org.elasticsearch.plugin</groupId>
<artifactId>x-pack-sql-jdbc</artifactId> <artifactId>x-pack-sql-jdbc</artifactId>
<version>6.4.2</version> <version>7.11.2</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>transport</artifactId>
<version>6.4.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.lucene</groupId> <groupId>org.apache.lucene</groupId>
<artifactId>lucene-core</artifactId> <artifactId>lucene-core</artifactId>
<version>7.4.0</version> <version>8.7.0</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
@@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch {
def findHouseMaxArea(String city, String region, String name, List alias) { def findHouseMaxArea(String city, String region, String name, List alias) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24) Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24)
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection
import com.mongodb.client.model.UpdateOptions import com.mongodb.client.model.UpdateOptions
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.bson.Document import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it-> DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it->
names << it.name names << it.name
} }
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch {
def findHouseCount(String city, String region, String name, List alias, int months) { def findHouseCount(String city, String region, String name, List alias, int months) {
Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months) Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months)
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject import com.mongodb.*
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement; import java.sql.Statement
/** /**
* *
* 历史均价计算 * 历史均价计算
@@ -27,7 +23,7 @@ import java.sql.Statement;
*/ */
@BatchOwner(Watcher.JiangYaHui) @BatchOwner(Watcher.JiangYaHui)
class HistoricalAverageRevaluationTask extends BaseBatch{ class HistoricalAverageRevaluationTask extends BaseBatch{
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat') public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat')
public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat') public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat')
@@ -1,6 +1,6 @@
package com.fangjia.script.avgprice package com.fangjia.script.avgprice
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
@@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.Jms import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.*
import com.mongodb.DBCollection;
import com.mongodb.DBCursor
import com.mongodb.DBObject
import com.mongodb.Mongo
import com.mongodb.Bytes
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement;
/** /**
* 无样本小区历史遗漏数据补充 * 无样本小区历史遗漏数据补充
@@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
@@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{
def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM"));
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -4,13 +4,13 @@ import com.fangjia.core.RegionData
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
class AddEsHouseField { class AddEsHouseField {
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.bson.types.ObjectId import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
@@ -63,7 +62,7 @@ class AddThirdPriceDistrict {
def province = null def province = null
def resAddrMap = [:] def resAddrMap = [:]
def filterEstateTypeMap = [:] def filterEstateTypeMap = [:]
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -1,11 +1,11 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import com.esms.common.util.CommonUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.huaban.analysis.jieba.JiebaSegmenter import com.huaban.analysis.jieba.JiebaSegmenter
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -485,7 +485,7 @@ class CrawlDataMatch {
} }
def findAllAddress(def cityName){ def findAllAddress(def cityName){
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,24 +1,19 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBObject
import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.SQLException import java.sql.SQLException
import java.sql.Statement
class EsHouseStat { class EsHouseStat {
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -1,12 +1,12 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
class EsSearchUtils { class EsSearchUtils {
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
@@ -20,7 +19,7 @@ class FZDataExport {
final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43")
def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43")
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
@@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{
public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl) public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl)
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -3,14 +3,14 @@ package com.fangjia.script.gj
import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
class RoomStat { class RoomStat {
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,30 +1,25 @@
package com.fangjia.script.gj package com.fangjia.script.gj
import com.fangjia.httpclient.UserAgents import com.fangjia.httpclient.UserAgents
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.JacksonUtil
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
import java.util.zip.GZIPInputStream import java.util.zip.GZIPInputStream
import static org.elasticsearch.index.query.QueryBuilders.idsQuery
class TradeDataMatch { class TradeDataMatch {
static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query" static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query"
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20)
def final AtomicInteger count_ = new AtomicInteger(0) def final AtomicInteger count_ = new AtomicInteger(0)
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
File diff suppressed because one or more lines are too long
@@ -1,24 +1,17 @@
package com.fangjia.script.gj.ccb package com.fangjia.script.gj.ccb
import com.fangjia.core.RegionData
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.FangJiaAreaMapUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils
import org.bson.types.ObjectId import org.bson.types.ObjectId
import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.BoolQueryBuilder
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.SimpleDateFormat
import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
import java.util.concurrent.ExecutorService import java.util.concurrent.ExecutorService
@@ -26,12 +19,11 @@ import java.util.concurrent.Executors
import java.util.concurrent.atomic.AtomicLong import java.util.concurrent.atomic.AtomicLong
import java.util.regex.Matcher import java.util.regex.Matcher
import java.util.regex.Pattern import java.util.regex.Pattern
import com.fangjia.script.util.FangJiaAreaMapUtils
class HouseExport { class HouseExport {
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement
class ExportData { class ExportData {
def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol(); def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol();
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.gj.EsSearchUtils
import com.fangjia.script.gj.district.FangjiaClientUtils import com.fangjia.script.gj.district.FangjiaClientUtils
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.*
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.GeoHelper
import com.fangjia.script.util.GeoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.bson.types.ObjectId import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
@@ -27,7 +22,7 @@ class DistrictWash {
public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5)
static def totalCount = new AtomicLong() static def totalCount = new AtomicLong()
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.ExcelUtils import com.fangjia.datastore.driver.DSCollection
import org.apache.commons.codec.digest.DigestUtils
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.regex.Matcher
import java.util.regex.Pattern
import net.sf.json.JSONObject
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DataStore import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils; import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils; import com.fangjia.script.common.DataHandleUtils
import com.fangjia.script.util.ExcelParser import com.fangjia.script.util.*
import com.fangjia.script.common.DataHandleUtils;
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.commons.io.FileUtils;
import org.bson.types.ObjectId
import com.fangjia.script.util.MongoUtils;
import com.mongodb.QueryOperators import com.mongodb.QueryOperators
import com.fangjia.script.util.OpenApiClient import net.sf.json.JSONObject
import org.apache.commons.codec.digest.DigestUtils
import java.util.Map.Entry; import org.apache.commons.io.FileUtils
import org.apache.commons.lang.StringUtils
import org.apache.poi.hssf.usermodel.HSSFWorkbook
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet
import java.text.DecimalFormat
import java.util.Map.Entry
import java.util.regex.Matcher
import java.util.regex.Pattern
class CZBlankDataExport { class CZBlankDataExport {
@@ -62,7 +52,7 @@ class CZBlankDataExport {
def directionMap = [1:"ÄÏ",2:"±±",3:"Äϱ±",4:"¶«",5:"Î÷",6:"¶«Î÷",7:"¶«ÄÏ",8:"Î÷ÄÏ",9:"¶«±±",10:"Î÷±±"] def directionMap = [1:"ÄÏ",2:"±±",3:"Äϱ±",4:"¶«",5:"Î÷",6:"¶«Î÷",7:"¶«ÄÏ",8:"Î÷ÄÏ",9:"¶«±±",10:"Î÷±±"]
public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
/** /**
* 江南银行线上数据交付 * 江南银行线上数据交付
@@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.crawl.CrawlCommandConsumerBatch
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ElasticUtils
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBList
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.client.transport.TransportClient
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.BatchUpdateException import java.sql.BatchUpdateException
import java.sql.ResultSet import java.sql.ResultSet
import java.text.NumberFormat
import java.util.Map.Entry
import java.util.concurrent.BlockingQueue
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
import java.util.concurrent.LinkedBlockingDeque import java.util.concurrent.LinkedBlockingDeque
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.ThreadPoolExecutor
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.regex.Pattern
@BatchOwner(Watcher.WeiLei) @BatchOwner(Watcher.WeiLei)
class HousingOverviewExportBatch extends BaseBatch { class HousingOverviewExportBatch extends BaseBatch {
static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>()) static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque<Runnable>())
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -1,6 +1,5 @@
package com.fangjia.script.jusfoun package com.fangjia.script.jusfoun
import com.alibaba.fastjson.JSONObject import com.alibaba.fastjson.JSONObject
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
@@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.SearchHit import org.elasticsearch.search.SearchHit
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import redis.clients.jedis.JedisPool
import redis.clients.jedis.JedisPoolConfig
import java.sql.* import java.sql.*
import java.text.DateFormat import java.text.DateFormat
@@ -56,7 +53,7 @@ class JusfounCommon {
public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver"); public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver"); public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver");
private static JdbcDataSource dataSource = null private static EsDataSource dataSource = null
private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>(); private static Map<String, BasicDBObject> _map_seaweed = new HashMap<>();
private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>(); private static Map<String, Map<String, DBObject>> mainCityMap = new HashMap<>();
@@ -390,7 +387,7 @@ class JusfounCommon {
return DataUtil.hanyuQuanpin(hanzi, '') return DataUtil.hanyuQuanpin(hanzi, '')
} }
static JdbcDataSource getESDataSource() { static EsDataSource getESDataSource() {
if (dataSource != null) { if (dataSource != null) {
return dataSource; return dataSource;
} else { } else {
@@ -399,7 +396,7 @@ class JusfounCommon {
return dataSource; return dataSource;
} }
try { try {
dataSource = new JdbcDataSource(); dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s"; String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address); dataSource.setUrl(address);
Properties connectionProperties = new Properties(); Properties connectionProperties = new Properties();
@@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math.stat.descriptive.moment.StandardDeviation import org.apache.commons.math.stat.descriptive.moment.StandardDeviation
import org.apache.commons.math3.stat.descriptive.moment.Mean import org.apache.commons.math3.stat.descriptive.moment.Mean
import org.bson.Document import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap
import static com.mongodb.client.model.Filters.* import static com.mongodb.client.model.Filters.*
def List find_rent_house_prices(String city, String region, String name, Date start, Date end){ def List find_rent_house_prices(String city, String region, String name, Date start, Date end){
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,20 +1,17 @@
package com.fangjia.script.log package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui) @BatchOwner(Watcher.JiangYaHui)
class MongoDBLogDisMonitorBatch extends BaseBatch { class MongoDBLogDisMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -1,20 +1,18 @@
package com.fangjia.script.log package com.fangjia.script.log
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
@BatchOwner(Watcher.JiangYaHui) @BatchOwner(Watcher.JiangYaHui)
class MongoDBLogMonitorBatch extends BaseBatch { class MongoDBLogMonitorBatch extends BaseBatch {
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c"
@@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
import java.util.concurrent.atomic.AtomicInteger import java.util.concurrent.atomic.AtomicInteger
@@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch {
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource() private static final EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -53,8 +53,10 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
void process(Object args) { void process(Object args) {
Map userInfo = getUserInfo() Map userInfo = getUserInfo()
def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3)) // def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3))
def endDate = DateUtil.truncateDay(new Date()) // def endDate = DateUtil.truncateDay(new Date())
def startDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210308"))
def endDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210314"))
BasicDBObject query = new BasicDBObject() BasicDBObject query = new BasicDBObject()
query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate)) query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate))
@@ -64,7 +66,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
long count = 0 long count = 0
long total = cursor.count() long total = cursor.count()
print("total="+total) print("total="+total)
BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia") BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("seawater.es.db.fangjia.com")
for(BasicDBObject log : cursor){ for(BasicDBObject log : cursor){
Map<String,Object> parameter_map = (Map) log.get("parameter_map") Map<String,Object> parameter_map = (Map) log.get("parameter_map")
if(parameter_map == null) { if(parameter_map == null) {
@@ -128,7 +130,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch {
cmd.put("user_name", user.user_name) cmd.put("user_name", user.user_name)
cmd.put("company_name", user.company) cmd.put("company_name", user.company)
} }
bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd)) bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd))
println(cmd) println(cmd)
count++ count++
if(count % 100 == 0){ if(count % 100 == 0){
@@ -1,50 +1,24 @@
package com.fangjia.script.trade package com.fangjia.script.trade
///µ¼Èë¼Û¸ñÖ¸Êý
import com.fangjia.script.BaseBatch;
import com.fangjia.core.util.DataUtil
import com.fangjia.script.BaseBatch
///导入价格指数
import com.fangjia.script.BatchOwner import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import com.fangjia.script.commData.CommonDatautils
import org.apache.commons.io.FileUtils; import com.fangjia.script.util.DateUtils
import org.apache.commons.lang.StringUtils;
import org.apache.poi.xssf.usermodel.XSSFCell
import org.apache.poi.xssf.usermodel.XSSFWorkbook
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.QueryOperators
import com.fangjia.script.util.DateUtils;
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection; import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import com.mongodb.Mongo import org.apache.commons.lang.StringUtils
import org.apache.commons.lang.math.RandomUtils
import groovy.json.JsonSlurper import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet import java.sql.ResultSet
import java.text.NumberFormat;
import java.util.regex.Matcher
import java.util.regex.Pattern
import com.fangjia.core.util.DBIterator
import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.StringUtil;
import com.fangjia.datastore.driver.DSCollection;
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.commData.CommonDatautils;
import com.ibm.icu.text.SimpleDateFormat;
import com.fangjia.storehouse.DBIterator
import com.fangjia.storehouse.DBIterator;
import com.fangjia.storehouse.StorehouseDB;
import org.apache.commons.lang.math.RandomUtils;
@BatchOwner(Watcher.GaoJuan) @BatchOwner(Watcher.GaoJuan)
class ImportIndexStats extends BaseBatch{ class ImportIndexStats extends BaseBatch{
//public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index'); //public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index');
@@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{
public static final def dingjiYear = 2016; public static final def dingjiYear = 2016;
private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
private static final JdbcDataSource dataSource = new JdbcDataSource() private static final EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,17 +1,12 @@
package com.fangjia.script.trade package com.fangjia.script.trade
import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtil
import com.fangjia.script.util.DateUtils import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.*
import com.mongodb.Bytes
import com.mongodb.DBCollection
import com.mongodb.DBObject
import com.mongodb.QueryOperators
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
@@ -22,7 +17,7 @@ class TradeDealPriceStat {
public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -3,7 +3,7 @@ package com.fangjia.script.usahouse
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.ResultSet import java.sql.ResultSet
import java.util.concurrent.CountDownLatch import java.util.concurrent.CountDownLatch
@@ -14,7 +14,7 @@ import java.util.regex.Pattern
class AddressMatch { class AddressMatch {
public static JdbcDataSource dataSource = new JdbcDataSource() public static EsDataSource dataSource = new EsDataSource()
private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
static { static {
dataSource.setUrl(address) dataSource.setUrl(address)
@@ -1,6 +1,6 @@
package com.fangjia.script.usahouse package com.fangjia.script.usahouse
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.httpclient.UserAgents import com.fangjia.httpclient.UserAgents
import com.fangjia.script.BaseBatch import com.fangjia.script.BaseBatch
@@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner
import com.fangjia.script.Watcher import com.fangjia.script.Watcher
import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.crawl.JQueryUtils
import com.fangjia.script.thirdPriceCrawl.CrawalUtil import com.fangjia.script.thirdPriceCrawl.CrawalUtil
import com.fangjia.script.util.HttpClient
import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.JacksonUtil
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import net.sf.json.JSON
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import org.jsoup.Jsoup import org.jsoup.Jsoup
import org.jsoup.nodes.Document import org.jsoup.nodes.Document
import org.jsoup.select.Elements import org.jsoup.select.Elements
@@ -1,69 +1,125 @@
package com.fangjia.script.util package com.fangjia.script.util
import org.apache.commons.lang3.SystemUtils
import org.apache.http.HttpHost
import org.apache.http.auth.AuthScope
import org.apache.http.auth.UsernamePasswordCredentials
import org.apache.http.client.CredentialsProvider
import org.apache.http.client.config.RequestConfig
import org.apache.http.impl.client.BasicCredentialsProvider
import org.apache.http.impl.nio.client.HttpAsyncClientBuilder
import org.elasticsearch.action.ActionListener
import org.elasticsearch.action.bulk.BackoffPolicy
import org.elasticsearch.action.bulk.BulkProcessor import org.elasticsearch.action.bulk.BulkProcessor
import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkRequest
import org.elasticsearch.action.bulk.BulkResponse import org.elasticsearch.action.bulk.BulkResponse
import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.client.RestClient
import org.elasticsearch.common.settings.Settings import org.elasticsearch.client.RestClientBuilder
import org.elasticsearch.common.transport.TransportAddress import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient import org.elasticsearch.client.RequestOptions
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.concurrent.TimeUnit import java.util.concurrent.TimeUnit
import java.util.function.BiConsumer
/** /**
* 功能说明:Elasticsearch工具类 * 功能说明:Elasticsearch工具类
*/ */
class ElasticsearchHelper { class ElasticsearchHelper {
static getClient(String host, String clusterName = "fangjia"){ static getEsDataSource(String host){
Settings.Builder builder = Settings.builder() EsDataSource dataSource = new EsDataSource ();
.put("cluster.name", clusterName) dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240");
.put("client.transport.sniff", false) Properties connectionProperties = new Properties();
.put("client.transport.ping_timeout", "60s") connectionProperties.setProperty("user","developer");
.put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3") connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3");
.put("xpack.security.transport.ssl.enabled", "true") dataSource.setProperties(connectionProperties);
if(SystemUtils.IS_OS_WINDOWS){ return dataSource;
builder }
.put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12")
.put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12") static executeQuery(String sql, Closure<Map> closure, String host = "192.168.0.216"){
}else{ EsDataSource dataSource = getEsDataSource(host)
builder Connection connection = dataSource.getConnection()
.put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12") Statement statement = connection.createStatement()
.put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12") ResultSet rs = statement.executeQuery(sql)
def columns = []
for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){
columns << rs.getMetaData().getColumnName(i)
} }
builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build() rs.getMetaData().getColumnCount()
Settings settings = builder.build() while(rs.next()){
return new PreBuiltXPackTransportClient(settings) def row = [:]
.addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300)) for (String col : columns){
row[col] = rs.getObject(col)
}
closure.call(row)
}
rs.close()
connection.close()
}
static RestHighLevelClient getClient(String host) {
RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http"));
builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() {
@Override
RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) {
requestConfigBuilder.setConnectionRequestTimeout(1000 * 60);
requestConfigBuilder.setSocketTimeout(1000 * 60);
return requestConfigBuilder;
}
})
builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
@Override
HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
httpAsyncClientBuilder.disableAuthCaching();
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456"));
httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return httpAsyncClientBuilder;
}
})
return new RestHighLevelClient(builder)
} }
/** /**
* 创建一个ES批处理器 * 创建一个ES批处理器
* @param clusterName
* @param host * @param host
* @return * @return
*/ */
static createBulkProcessor( String host, String clusterName){ static createBulkProcessor(String host) {
TransportClient client = getClient(host, clusterName) BulkProcessor.Listener listener = new BulkProcessor.Listener() {
BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() {
@Override @Override
void beforeBulk(long executionId, BulkRequest request) { void beforeBulk(long executionId, BulkRequest request) {
} }
@Override @Override
void afterBulk(long executionId, BulkRequest request, BulkResponse response) { void afterBulk(long executionId, BulkRequest request,
BulkResponse response) {
} }
@Override @Override
void afterBulk(long executionId, BulkRequest request, Throwable failure) { void afterBulk(long executionId, BulkRequest request,
failure.printStackTrace() Throwable failure) {
} }
}).setBulkActions(10000).setConcurrentRequests(20).build() };
RestHighLevelClient client = getClient(host);
return bulkProcessor return BulkProcessor.builder(new BiConsumer<BulkRequest, ActionListener<BulkResponse>>() {
@Override
void accept(BulkRequest bulkRequest, ActionListener<BulkResponse> bulkResponseActionListener) {
client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener)
}
}, listener).setBulkActions(10000)
.setConcurrentRequests(20)
.setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3))
.setFlushInterval(TimeValue.timeValueSeconds(10L)).build();
} }
/** /**
@@ -71,14 +127,13 @@ class ElasticsearchHelper {
* @param * @param
* @return * @return
*/ */
static closeBulkProcessor(BulkProcessor bulkProcessor){ static closeBulkProcessor(BulkProcessor bulkProcessor) {
bulkProcessor.awaitClose(10, TimeUnit.MINUTES) bulkProcessor.awaitClose(10, TimeUnit.DAYS)
bulkProcessor.flush() bulkProcessor.flush()
bulkProcessor.close() bulkProcessor.close()
} }
} }
@@ -1,6 +1,6 @@
package com.fangjia.script.weilei package com.fangjia.script.weilei
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.Bytes import com.mongodb.Bytes
import com.mongodb.DBCollection import com.mongodb.DBCollection
@@ -8,7 +8,7 @@ import com.mongodb.DBCursor
import com.mongodb.DBObject import com.mongodb.DBObject
import net.sf.json.JSONObject import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -16,7 +16,7 @@ import java.sql.SQLException
import java.sql.Statement import java.sql.Statement
import java.text.SimpleDateFormat import java.text.SimpleDateFormat
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s"; String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s";
dataSource.setUrl(address); dataSource.setUrl(address);
Properties connectionProperties = new Properties(); Properties connectionProperties = new Properties();
@@ -1,31 +1,16 @@
import com.fangjia.core.Constants
import com.fangjia.core.RegionData
import com.fangjia.core.util.Mongo3Utils import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.Jms
import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject
import com.mongodb.CommandResult
import com.mongodb.DB
import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection import com.mongodb.client.MongoCollection
import org.bson.Document import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement import java.sql.Statement
import java.util.function.Consumer import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*; import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"] def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"]
def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"] def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"]
@@ -43,7 +28,7 @@ cityList.each { def city ->
} }
}) })
roots.each { def root -> roots.each { def root ->
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,23 +1,12 @@
package com.fangjia.script.wenyan.address package com.fangjia.script.wenyan.address
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.yinjihaun.util.QQKeyPool
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.apache.lucene.index.Term
import org.apache.lucene.search.BooleanClause
import org.apache.lucene.search.BooleanQuery
import org.apache.lucene.search.TermQuery
import org.apache.solr.client.solrj.util.ClientUtils import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -44,7 +33,7 @@ import java.sql.Statement
// data << (k.split("_") as List) // data << (k.split("_") as List)
//} //}
//ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û") //ExcelUtils.write("D:\\ÎÞ¾ù¼ÛÊý¾Ý.xlsx", data, "½á¹û")
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s";
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,20 +1,20 @@
package com.fangjia.script.wenyan.avgPrice package com.fangjia.script.wenyan.avgPrice
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.apache.solr.client.solrj.util.ClientUtils import org.apache.solr.client.solrj.util.ClientUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement import java.sql.Statement
def findEstateType(String city, String region, String name, List alias) { def findEstateType(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils
import org.apache.commons.math3.ml.clustering.CentroidCluster import org.apache.commons.math3.ml.clustering.CentroidCluster
import org.apache.commons.math3.ml.clustering.DoublePoint import org.apache.commons.math3.ml.clustering.DoublePoint
import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement import java.sql.Statement
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017) DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017)
def findHouse(BasicDBObject district){ def findHouse(BasicDBObject district){
@@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){
sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ") sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ")
sqlBuffer.append(" ORDER BY p_date desc") sqlBuffer.append(" ORDER BY p_date desc")
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s"; String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s";
dataSource.setUrl(address); dataSource.setUrl(address);
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,13 +1,13 @@
package com.fangjia.script.wenyan.seawater package com.fangjia.script.wenyan.seawater
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement import java.sql.Statement
JdbcDataSource dataSource = new JdbcDataSource(); EsDataSource dataSource = new EsDataSource();
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,20 +1,12 @@
package com.fangjia.script.wenyan.temp package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
import com.mongodb.client.MongoCollection import com.mongodb.client.MongoCollection
import org.bson.Document import org.bson.Document
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection
import java.sql.ResultSet
import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.and
import static com.mongodb.client.model.Filters.eq import static com.mongodb.client.model.Filters.eq
// //
//def data = [["城市", "区域", "小区名", "2018", "全部"]] //def data = [["城市", "区域", "小区名", "2018", "全部"]]
//cityList = ["上海"] //cityList = ["上海"]
@@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq
//HttpRequest httpRequest = new HttpRequest() //HttpRequest httpRequest = new HttpRequest()
//println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8") //println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8")
import org.bson.Document
import com.mongodb.client.MongoCollection
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config") MongoCollection<Document> AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config")
AVERAGER_CONFIG.find().each {Document doc-> AVERAGER_CONFIG.find().each {Document doc->
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.* import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20201101", "yyyyMMdd") def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20201201", "yyyyMMdd") def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance() Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate) calendar.setTime(startDate)
def count = 0 def count = 0
@@ -1,19 +1,13 @@
package com.fangjia.script.wenyan.temp package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.Mongo
import com.mongodb.client.MongoCollection
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.bson.Document import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -61,7 +55,7 @@ import java.sql.Statement
//println(bigMarkupCount) //println(bigMarkupCount)
def countHouses(String city, String region, String name){ def countHouses(String city, String region, String name){
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,28 +1,16 @@
package com.fangjia.script.wenyan.temp package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCollection
import com.fangjia.datastore.driver.DataStore
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.bson.Document import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import com.mongodb.client.MongoCollection
import org.bson.types.ObjectId
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement import java.sql.Statement
import java.util.function.Consumer
import static com.mongodb.client.model.Filters.*
import com.fangjia.core.util.Mongo3Utils
def poiData = Collections.synchronizedList([]) def poiData = Collections.synchronizedList([])
// //
//ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item -> //ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item ->
@@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([])
// println(item) // println(item)
//},1) //},1)
def findHouseCount(String city, String region, String name, List alias) { def findHouseCount(String city, String region, String name, List alias) {
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -11,8 +11,8 @@ import org.bson.Document
import static com.mongodb.client.model.Filters.* import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210201", "yyyyMMdd") def startDate = DateUtil.parseDate("20210301", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210301", "yyyyMMdd") def endDate = DateUtil.parseDate("20210401", "yyyyMMdd")
Calendar calendar = Calendar.getInstance() Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate) calendar.setTime(startDate)
def count = 0 def count = 0
@@ -1,11 +1,17 @@
import com.fangjia.core.RegionData import com.fangjia.core.RegionData
import com.fangjia.core.util.BaiDuApiUtils
import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DataUtil
import com.fangjia.core.util.DateUtil import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.GeoUtils
import com.fangjia.core.util.GeocodeUtils
import com.fangjia.core.util.HttpRequest import com.fangjia.core.util.HttpRequest
import com.fangjia.datastore.driver.DSCursor import com.fangjia.datastore.driver.DSCursor
import com.fangjia.datastore.driver.FacetResult import com.fangjia.datastore.driver.FacetResult
import com.fangjia.script.util.BaiduMapUtils
import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
//import com.fangjia.script.util.ExcelUtils //import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient import com.fangjia.script.util.OpenApiClient
@@ -132,4 +138,70 @@ import org.apache.commons.lang3.StringUtils
// item["E"] = RegionData.formatRegion(item["B"], item["C"]) // item["E"] = RegionData.formatRegion(item["B"], item["C"])
// println(item) // println(item)
//}) //})
CommonUtils.reFitAverage("常州","新北","龙城花园", "普通住宅") //def border = [13511506.7,3639168.61,13511516.8,3639404.86,13511515.27,3639416.92,13511738.21,3639384.94,13512007.72,3639336.35,13511984.92,3639074.96,13511722.75,3639121.08,13511506.7,3639168.61]
//def coords = []
//for(int i=0; i < border.size() - 1; i=i+2){
// coords << "${border[i]},${border[i+1]}"
//}
//def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
//println(json)
//def points = []
//for(JSONObject it : json.result){
// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
// points << "${geo.lng},${geo.lat}"
// println(" new AMap.LngLat(${geo.lng},${geo.lat}),")
//}
DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020)
BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo->
def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List
def coords = []
def baidu_points = []
def gaode_points = []
for(int i=0; i < border.size() - 1; i=i+2){
coords << "${border[i]},${border[i+1]}"
if(coords.size() > 50){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
coords.clear()
}
}
if(coords.size() > 0){
def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5)
for(JSONObject it : json.result){
baidu_points << "${it.x},${it.y}"
def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y)
gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}"
}
}
dbo.format_city = RegionData.formatCity(dbo.city)
dbo.baidu_border = baidu_points.join(";")
dbo.gaode_border = gaode_points.join(";")
BAIDUMAP_BORDER_COLL.save(dbo)
println(dbo)
}
//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\È˿ڲâÊÔÊý¾Ý.xlsx",{List item->
// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1))
// if(seaweed){
// item["A"] = seaweed._id.toString()
// def names = [item['D']]
// if(seaweed.alias){
// names.addAll(seaweed.alias)
// }
// names.unique()
// if(seaweed.lng){
// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6)
// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6)
// }
// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names)))
// if(one){
// item["I"] = one.baidu_border
// }
// println(item)
// }
//})
@@ -669,7 +669,6 @@ def findRentInfoByRoom(String city, String region){
def findRent(def city, def region, def name){ def findRent(def city, def region, def name){
DBCollection DISTRICT_STAT_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017)
def query = new BasicDBObject() def query = new BasicDBObject()
query.city = city query.city = city
query.region = region query.region = region
@@ -1,19 +1,8 @@
package com.fangjia.script.wenyan.temp package com.fangjia.script.wenyan.temp
import com.fangjia.core.util.DateUtil
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONArray
import net.sf.json.JSONObject
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection import com.fangjia.script.util.ExcelUtils
import java.sql.ResultSet import com.mongodb.BasicDBObject
import java.sql.Statement
//DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com") //DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com")
////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\下载记录20200818-结果 - 副本.xlsx",{List item-> ////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\下载记录20200818-结果 - 副本.xlsx",{List item->
@@ -1,23 +1,9 @@
package com.fangjia.script.wenyan.temp package com.fangjia.script.wenyan.temp
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DataUtil
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.FangJiaAreaMapUtils
import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.fangjia.script.util.ProvinceData
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection
import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.Connection import com.fangjia.script.util.ExcelUtils
import java.sql.ResultSet import com.fangjia.script.util.MongoUtils
import java.sql.Statement import com.mongodb.DBCollection
// //
//ExcelUtils.multiThreadingModify("D:\\数据\\建行学校数据\\上海学校_0811.xls",{List item-> //ExcelUtils.multiThreadingModify("D:\\数据\\建行学校数据\\上海学校_0811.xls",{List item->
@@ -5,13 +5,12 @@ import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
import java.sql.Statement import java.sql.Statement
//def data = [] //def data = []
//data << ["城市", "区域", "小区名", "近一月", "近三月", "近6月", "近一年", "户数"] //data << ["城市", "区域", "小区名", "近一月", "近三月", "近6月", "近一年", "户数"]
//DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("上海")).each {BasicDBObject district-> //DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("上海")).each {BasicDBObject district->
@@ -36,7 +35,7 @@ import java.sql.Statement
//ExcelUtils.write("D:\\上海成交统计.xlsx", data, "成交数据") //ExcelUtils.write("D:\\上海成交统计.xlsx", data, "成交数据")
def find_rent_house_prices(String city, String region, String name){ def find_rent_house_prices(String city, String region, String name){
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -80,7 +79,7 @@ def find_rent_house_prices(String city, String region, String name){
} }
ExcelUtils.multiThreadingModify("D:\\上海成交统计.xlsx", {List item-> ExcelUtils.multiThreadingModify("D:\\上海成交统计.xlsx", {List item->
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,13 +1,12 @@
package com.fangjia.script.wenyan.temp package com.fangjia.script.wenyan.temp
import com.fangjia.script.util.* import com.fangjia.script.util.*
import com.mongodb.BasicDBObject import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import com.mongodb.DBObject import com.mongodb.DBObject
import net.sf.json.JSONObject import net.sf.json.JSONObject
import org.apache.commons.lang.StringUtils import org.apache.commons.lang.StringUtils
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import java.sql.Connection import java.sql.Connection
import java.sql.ResultSet import java.sql.ResultSet
@@ -16,7 +15,7 @@ import java.sql.Statement
DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43") DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43")
def String hasElevator(BasicDBObject seaweed){ def String hasElevator(BasicDBObject seaweed){
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
@@ -1,40 +1,7 @@
package com.fangjia.script.wenyan.temp package com.fangjia.script.wenyan.temp
import com.fangjia.script.util.ElasticsearchHelper
import com.fangjia.core.util.DateUtil ElasticsearchHelper.executeQuery("select * from seawater where p_date > 20210407 limit 10",{Map row->
import com.fangjia.script.util.CommonUtils println(row.name)
import com.fangjia.script.util.DbUtils }, "192.168.0.216")
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\新增小区.xlsx", 0, 0).each {List item->
def keys = new BasicDBObject()
keys.city = 1
keys.region = 1
keys.name = 1
keys.estate_type2 = 1
def query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"])
query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
if(!DbUtils.findDistrictStat(dbo, new Date())){
CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
println(dbo)
}
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
// println(dbo)
}
}
//["佛山","济南","北京","广州","深圳"].each {def city->
// def keys = new BasicDBObject()
// keys.city = 1
// keys.region = 1
// keys.name = 1
// keys.estate_type2 = 1
// def query = DbUtils.buildSeaweedQuery(city)
// query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1)))
// DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo->
// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2)
// println(dbo)
// }
//}
@@ -1,20 +1,14 @@
package com.fangjia.script.yinjihaun.price package com.fangjia.script.yinjihaun.price
import com.fangjia.core.util.HttpRequest
import com.fangjia.script.commData.CommonDatautils
import com.fangjia.script.util.DateUtils
import com.fangjia.script.util.ExcelUtils
import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.MongoUtils
import com.fangjia.script.util.OpenApiClient
import com.mongodb.BasicDBObject
import com.mongodb.DBCollection import com.mongodb.DBCollection
import org.apache.commons.lang.StringUtils import org.elasticsearch.xpack.sql.jdbc.EsDataSource
import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource
import java.sql.ResultSet import java.sql.ResultSet
String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"
JdbcDataSource dataSource = new JdbcDataSource() EsDataSource dataSource = new EsDataSource()
dataSource.setUrl(address) dataSource.setUrl(address)
Properties connectionProperties = new Properties() Properties connectionProperties = new Properties()
connectionProperties.put("user", "developer") connectionProperties.put("user", "developer")