From 32d1f77ec7f100eba63cc4feb2ca94aa3eaec75b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Thu, 8 Apr 2021 13:32:52 +0800 Subject: [PATCH] =?UTF-8?q?=E5=8D=87=E7=BA=A7Es=E9=A9=B1=E5=8A=A8=E7=89=88?= =?UTF-8?q?=E6=9C=AC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../com/fangjia/core/util/BaiDuApiUtils.java | 33 +++++ projects/fangjia-script/pom.xml | 29 +--- .../ana/DistrictPriceRatingBatch.groovy | 4 +- .../analy/DistrictFeatureAnalyBatch.groovy | 6 +- .../HistoricalAverageRevaluationTask.groovy | 14 +- .../NoSampleAverageRevaluationTask.groovy | 15 +- .../script/fuzhou/HouseExportBatch.groovy | 4 +- .../fangjia/script/gj/AddEsHouseField.groovy | 4 +- .../script/gj/AddThirdPriceDistrict.groovy | 5 +- .../fangjia/script/gj/CrawlDataMatch.groovy | 6 +- .../com/fangjia/script/gj/EsHouseStat.groovy | 11 +- .../fangjia/script/gj/EsSearchUtils.groovy | 4 +- .../com/fangjia/script/gj/FZDataExport.groovy | 5 +- .../script/gj/NoHouseDistrictStatBatch.groovy | 5 +- .../com/fangjia/script/gj/RoomStat.groovy | 4 +- .../fangjia/script/gj/TradeDataMatch.groovy | 9 +- .../gj/VagueAddressLocationBatch.groovy | 7 +- .../fangjia/script/gj/ccb/HouseExport.groovy | 16 +-- .../script/gj/district/ExportData.groovy | 6 +- .../gj/districtwash/DistrictWash.groovy | 11 +- .../jiangnanblank/CZBlankDataExport.groovy | 44 +++--- .../JiangNanBlankHousesDataBatch.groovy | 1 - .../jusfoun/HousingOverviewExportBatch.groovy | 19 +-- .../script/jusfoun/JusfounCommon.groovy | 11 +- .../lixue/dataexport/AddressComExport.groovy | 4 +- .../log/MongoDBLogDisMonitorBatch.groovy | 13 +- .../script/log/MongoDBLogMonitorBatch.groovy | 14 +- .../rent/DistrictHouseTypeAnalyzeBatch.groovy | 5 +- .../sync/AnalysisOpenUriLogSyncBatch.groovy | 10 +- .../script/trade/ImportIndexStats.groovy | 48 ++----- .../script/trade/TradeDealPriceStat.groovy | 11 +- .../script/usahouse/AddressMatch.groovy | 4 +- .../usahouse/NewZealandWaiJuCrawl.groovy | 5 +- .../script/util/ElasticsearchHelper.groovy | 129 +++++++++++++----- .../script/weilei/Seabird_Seawater.groovy | 6 +- .../com/fangjia/script/wenyan/Test3.groovy | 23 +--- .../script/wenyan/address/Test1.groovy | 17 +-- .../script/wenyan/avgPrice/Test4.groovy | 6 +- .../script/wenyan/rent/RentStat.groovy | 9 +- .../seawater/StatsSeawaterDirection.groovy | 4 +- .../seawater/StatsSeawaterFitment.groovy | 4 +- .../script/wenyan/temp/ChangZou.groovy | 16 +-- .../fangjia/script/wenyan/temp/QinLin.groovy | 4 +- .../fangjia/script/wenyan/temp/SanSha.groovy | 12 +- .../fangjia/script/wenyan/temp/Seaweed.groovy | 18 +-- .../wenyan/temp/SiChuanJingCheng.groovy | 4 +- .../fangjia/script/wenyan/temp/Test.groovy | 74 +++++++++- .../fangjia/script/wenyan/temp/Test11.groovy | 1 - .../fangjia/script/wenyan/temp/Test16.groovy | 15 +- .../fangjia/script/wenyan/temp/Test20.groovy | 20 +-- .../fangjia/script/wenyan/temp/Test8.groovy | 7 +- .../fangjia/script/wenyan/temp/XingYe.groovy | 5 +- .../script/wenyan/temp/xiaoqugeo.groovy | 41 +----- .../yinjihaun/price/SaveDistrict.groovy | 12 +- 54 files changed, 361 insertions(+), 453 deletions(-) diff --git a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java index 5c5df16fe..e075fbd32 100644 --- a/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java +++ b/projects/fangjia-core/src/main/java/com/fangjia/core/util/BaiDuApiUtils.java @@ -277,6 +277,39 @@ public class BaiDuApiUtils { } return null; } + + /** + * + * @param coords + * @param from + * @return + */ + public static JSONObject geoconv(String coords,int from){ + Map params = new HashMap(); + params.put("output", "json"); + params.put("ak", AK); + params.put("coords", coords); + params.put("from", String.valueOf(from)); + HttpRequest request = new HttpRequest(); + try{ + String result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); + JSONObject json = JSONObject.fromObject(result); + while(!(json == null || json.containsKey("result") || json.getJSONArray("result").size() == 0)){ + changeAK(); + result = request.get("http://api.map.baidu.com/geoconv/v1/",params,"UTF-8"); + json = JSONObject.fromObject(result); + Thread.sleep(1000 * 60); + } + if(json.getInt("status") >= 300){ + changeAK(); + json = geoconv(coords, from); + } + return json.getJSONArray("result").getJSONObject(0); + }catch (Exception e){ + e.printStackTrace(); + } + return null; + } public static JSONObject geoconv(String coords, int from, int to){ Map params = new HashMap(); diff --git a/projects/fangjia-script/pom.xml b/projects/fangjia-script/pom.xml index cd0782adc..bb7959032 100644 --- a/projects/fangjia-script/pom.xml +++ b/projects/fangjia-script/pom.xml @@ -17,8 +17,8 @@ maven-compiler-plugin 2.3.2 - 1.7 - 1.7 + 1.8 + 1.8 GBK @@ -179,7 +179,7 @@ org.apache.httpcomponents httpclient - 4.4.1 + 4.5.10 org.apache.ant @@ -315,16 +315,6 @@ metadata-extractor 2.4.0 - - org.apache.http - httpasyncclient - 4.0 - - - org.apache.http - httpcore-nio - 4.1.1 - org.apache.sanselan sanselan @@ -448,23 +438,18 @@ org.elasticsearch.client - x-pack-transport - 6.4.2 + elasticsearch-rest-high-level-client + 7.11.2 org.elasticsearch.plugin x-pack-sql-jdbc - 6.4.2 - - - org.elasticsearch.client - transport - 6.4.2 + 7.11.2 org.apache.lucene lucene-core - 7.4.0 + 8.7.0 org.apache.commons diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictPriceRatingBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictPriceRatingBatch.groovy index 63cccd59e..f34dc1923 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictPriceRatingBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/ana/DistrictPriceRatingBatch.groovy @@ -13,7 +13,7 @@ import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.lang.StringUtils import org.apache.solr.client.solrj.util.ClientUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -147,7 +147,7 @@ class DistrictPriceRatingBatch extends BaseBatch { def findHouseMaxArea(String city, String region, String name, List alias) { Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -24) - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/analy/DistrictFeatureAnalyBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/analy/DistrictFeatureAnalyBatch.groovy index 8282a2a69..aca381ea3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/analy/DistrictFeatureAnalyBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/analy/DistrictFeatureAnalyBatch.groovy @@ -13,7 +13,7 @@ import com.mongodb.client.MongoCollection import com.mongodb.client.model.UpdateOptions import org.apache.commons.lang.StringUtils import org.bson.Document -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -94,7 +94,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch { DbUtils.seaweed.find(query, new BasicDBObject("name", 1)).each {BasicDBObject it-> names << it.name } - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -129,7 +129,7 @@ class DistrictFeatureAnalyBatch extends BaseBatch { def findHouseCount(String city, String region, String name, List alias, int months) { Date startDate = DateUtil.add(new Date(), Calendar.MONTH, -months) - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/HistoricalAverageRevaluationTask.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/HistoricalAverageRevaluationTask.groovy index 56c47b60b..c62639177 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/HistoricalAverageRevaluationTask.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/HistoricalAverageRevaluationTask.groovy @@ -7,18 +7,14 @@ import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DbUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection; -import com.mongodb.DBCursor -import com.mongodb.DBObject -import com.mongodb.Mongo -import com.mongodb.Bytes +import com.mongodb.* import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet -import java.sql.Statement; +import java.sql.Statement + /** * * 历史均价计算 @@ -27,7 +23,7 @@ import java.sql.Statement; */ @BatchOwner(Watcher.JiangYaHui) class HistoricalAverageRevaluationTask extends BaseBatch{ - JdbcDataSource dataSource = new JdbcDataSource(); + EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; public final static DBCollection DISTRICT_STAT_COLL = new Mongo("mongo.db.fangjia.com", 27017).getDB('fangjia_stat').getCollection('district_stat') public final static DBCollection NOPRICE_DISTRICT_STAT_COLL = new Mongo("log.mongo.db.fangjia.com", 52002).getDB('fangjia_log').getCollection('noprice_district_stat') diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/NoSampleAverageRevaluationTask.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/NoSampleAverageRevaluationTask.groovy index ce9352018..e093d6d29 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/NoSampleAverageRevaluationTask.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/avgprice/NoSampleAverageRevaluationTask.groovy @@ -1,6 +1,6 @@ package com.fangjia.script.avgprice -import com.fangjia.core.RegionData + import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner @@ -9,18 +9,7 @@ import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.Jms import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection; -import com.mongodb.DBCursor -import com.mongodb.DBObject -import com.mongodb.Mongo -import com.mongodb.Bytes -import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource - -import java.sql.Connection -import java.sql.ResultSet -import java.sql.Statement; +import com.mongodb.* /** * 无样本小区历史遗漏数据补充 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy index 9b6ac0db7..0ff84d43e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/fuzhou/HouseExportBatch.groovy @@ -17,7 +17,7 @@ import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.SearchHit -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet import java.util.concurrent.ConcurrentHashMap @@ -37,7 +37,7 @@ class HouseExportBatch extends BaseBatch{ def curDate = Integer.parseInt(DateUtils.addMonth(DateUtils.formatDateByPattern(new Date(), "yyyyMM"), -1, "yyyyMM")); public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddEsHouseField.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddEsHouseField.groovy index d66b80555..9af2b800d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddEsHouseField.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddEsHouseField.groovy @@ -4,13 +4,13 @@ import com.fangjia.core.RegionData import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DbUtils import com.mongodb.BasicDBObject -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet class AddEsHouseField { - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy index 4983deb32..aece3ef15 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/AddThirdPriceDistrict.groovy @@ -9,10 +9,9 @@ import com.fangjia.script.util.ExcelUtils import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import org.bson.types.ObjectId -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet -import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger @@ -63,7 +62,7 @@ class AddThirdPriceDistrict { def province = null def resAddrMap = [:] def filterEstateTypeMap = [:] - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CrawlDataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CrawlDataMatch.groovy index 1a8b400fe..1aed12e90 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CrawlDataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/CrawlDataMatch.groovy @@ -1,11 +1,11 @@ package com.fangjia.script.gj -import com.esms.common.util.CommonUtil + import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.ExcelUtils import com.huaban.analysis.jieba.JiebaSegmenter -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -485,7 +485,7 @@ class CrawlDataMatch { } def findAllAddress(def cityName){ - JdbcDataSource dataSource = new JdbcDataSource(); + EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy index be42a6ee6..401002889 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsHouseStat.groovy @@ -1,24 +1,19 @@ package com.fangjia.script.gj -import com.fangjia.script.commData.CommonDatautils + import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.ExcelUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBObject -import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource -import java.sql.Connection import java.sql.ResultSet import java.sql.SQLException -import java.sql.Statement class EsHouseStat { - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy index b6bf9f817..dc7821343 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/EsSearchUtils.groovy @@ -1,12 +1,12 @@ package com.fangjia.script.gj -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet class EsSearchUtils { - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FZDataExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FZDataExport.groovy index 953e21be5..6c33b2553 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FZDataExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/FZDataExport.groovy @@ -8,10 +8,9 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import org.apache.commons.lang.StringUtils import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet @@ -20,7 +19,7 @@ class FZDataExport { final DBCollection DEAL_PRICE = MongoUtils.getCollectionByName("fangjia", "deal_price", "192.168.0.43") def static final DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "192.168.0.43") - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/NoHouseDistrictStatBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/NoHouseDistrictStatBatch.groovy index 01dc067cc..be975fe6e 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/NoHouseDistrictStatBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/NoHouseDistrictStatBatch.groovy @@ -7,11 +7,10 @@ import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject -import com.mongodb.Bytes import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet import java.util.concurrent.CountDownLatch @@ -27,7 +26,7 @@ class NoHouseDistrictStatBatch extends BaseBatch{ public static final DBCollection AVERAGER_CONFIG_COLL = MongoUtils.getCollectionByName2("fangjia", "averager_config", dbUrl) public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RoomStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RoomStat.groovy index 1d1d00222..e83135e20 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RoomStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/RoomStat.groovy @@ -3,14 +3,14 @@ package com.fangjia.script.gj import com.fangjia.script.common.DataHandleUtils import com.fangjia.script.util.ExcelUtils import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet class RoomStat { public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeDataMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeDataMatch.groovy index ca7c6d627..c395a8a2a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeDataMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/TradeDataMatch.groovy @@ -1,30 +1,25 @@ package com.fangjia.script.gj import com.fangjia.httpclient.UserAgents -import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.ElasticUtils import com.fangjia.script.util.JacksonUtil import org.apache.commons.lang.StringUtils import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource -import java.sql.ResultSet -import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicInteger import java.util.zip.GZIPInputStream -import static org.elasticsearch.index.query.QueryBuilders.idsQuery - class TradeDataMatch { static def esf_queryUrl = "http://192.168.0.45:9200/esf_deal_price_lansi/_update_by_query" public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(20) def final AtomicInteger count_ = new AtomicInteger(0) public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy index ac61103e1..774747a5f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/VagueAddressLocationBatch.groovy @@ -1,16 +1,15 @@ package com.fangjia.script.gj -import com.fangjia.core.util.DataUtil + import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.common.DataHandleUtils -import com.fangjia.script.lixue.util.DBUtils import com.fangjia.script.util.DateUtils import com.fangjia.script.util.GeoHelper import com.mongodb.BasicDBObject -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet import java.util.regex.Matcher @@ -22,7 +21,7 @@ import java.util.regex.Pattern @BatchOwner(Watcher.GaoJuan) class VagueAddressLocationBatch extends BaseBatch { - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" def codeMap = ["北京": "131", "天津": "332", "石家庄": "150", "唐山": "265", "秦皇岛": "148", "邯郸": "151", "邢台": "266", "保定": "307", "张家口": "264", "承德": "207", "沧州": "149", "廊坊": "191", "衡水": "208", "太原": "176", "大同": "355", "阳泉": "357", "长治": "356", "晋城": "290", "朔州": "237", "晋中": "238", "运城": "328", "忻州": "367", "临汾": "368", "吕梁": "327", "呼和浩特": "321", "包头": "229", "乌海": "123", "赤峰": "297", "通辽": "64", "鄂尔多斯": "283", "呼伦贝尔": "61", "巴彦淖尔": "169", "乌兰察布": "168", "兴安盟": "62", "锡林郭勒盟": "63", "阿拉善盟": "230", "沈阳": "58", "大连": "167", "鞍山": "320", "抚顺": "184", "本溪": "227", "丹东": "282", "锦州": "166", "营口": "281", "阜新": "59", "辽阳": "351", "盘锦": "228", "铁岭": "60", "朝阳": "280", "葫芦岛": "319", "长春": "53", "吉林市": "55", "四平": "56", "辽源": "183", "通化": "165", "白山": "57", "松原": "52", "白城": "51", "延边朝鲜族自治州": "54", "哈尔滨": "48", "齐齐哈尔": "41", "鸡西": "46", "鹤岗": "43", "双鸭山": "45", "大庆": "50", "伊春": "40", "佳木斯": "42", "七台河": "47", "牡丹江": "49", "黑河": "39", "绥化": "44", "大兴安岭地区": "38", "上海": "289", "南京": "315", "无锡": "317", "徐州": "316", "常州": "348", "苏州": "224", "南通": "161", "连云港": "347", "淮安": "162", "盐城": "223", "扬州": "346", "镇江": "160", "泰州": "276", "宿迁": "277", "杭州": "179", "宁波": "180", "温州": "178", "嘉兴": "334", "湖州": "294", "绍兴": "293", "金华": "333", "衢州": "243", "舟山": "245", "台州": "244", "丽水": "292", "合肥": "127", "芜湖": "129", "蚌埠": "126", "淮南": "250", "马鞍山": "358", "淮北": "253", "铜陵": "337", "安庆": "130", "黄山": "252", "滁州": "189", "阜阳": "128", "宿州": "370", "巢湖": "251", "六安": "298", "亳州": "188", "池州": "299", "宣城": "190", "福州": "300", "厦门": "194", "莆田": "195", "三明": "254", "泉州": "134", "漳州": "255", "南平": "133", "龙岩": "193", "宁德": "192", "南昌": "163", "景德镇": "225", "萍乡": "350", "九江": "349", "新余": "164", "鹰潭": "279", "赣州": "365", "吉安": "318", "宜春": "278", "抚州": "226", "上饶": "364", "济南": "288", "青岛": "236", "淄博": "354", "枣庄": "172", "东营": "174", "烟台": "326", "潍坊": "287", "济宁": "286", "泰安": "325", "威海": "175", "日照": "173", "莱芜": "124", "临沂": "234", "德州": "372", "聊城": "366", "滨州": "235", "菏泽": "353", "郑州": "268", "开封": "210", "洛阳": "153", "平顶山": "213", "安阳": "267", "鹤壁": "215", "新乡": "152", "焦作": "211", "濮阳": "209", "许昌": "155", "漯河": "344", "三门峡": "212", "南阳": "309", "商丘": "154", "信阳": "214", "周口": "308", "驻马店": "269", "武汉": "218", "黄石": "311", "十堰": "216", "宜昌": "270", "襄阳": "156", "鄂州": "122", "荆门": "217", "孝感": "310", "荆州": "157", "黄冈": "271", "咸宁": "362", "随州": "371", "恩施土家族苗族自治州": "373", "仙桃": "1713", "潜江": "1293", "天门": "2654", "神农架林区": "2734", "长沙": "158", "株洲": "222", "湘潭": "313", "衡阳": "159", "邵阳": "273", "岳阳": "220", "常德": "219", "张家界": "312", "益阳": "272", "郴州": "275", "永州": "314", "怀化": "363", "娄底": "221", "湘西土家族苗族自治州": "274", "广州": "257", "韶关": "137", "深圳": "340", "珠海": "140", "汕头": "303", "佛山": "138", "江门": "302", "湛江": "198", "茂名": "139", "肇庆": "338", "惠州": "301", "梅州": "141", "汕尾": "339", "河源": "200", "阳江": "199", "清远": "197", "东莞": "119", "中山": "187", "潮州": "201", "揭阳": "259", "云浮": "258", "南宁": "261", "柳州": "305", "桂林": "142", "梧州": "304", "北海": "295", "防城港": "204", "钦州": "145", "贵港": "341", "玉林": "361", "百色": "203", "贺州": "260", "河池": "143", "来宾": "202", "崇左": "144", "海口": "125", "三亚": "121", "五指山": "1644", "琼海": "2358", "儋州": "1215", "文昌": "2758", "万宁": "1216", "东方": "2634", "定安": "1214", "屯昌": "1641", "澄迈": "2757", "临高": "2033", "白沙黎族自治": "2359", "昌江黎族自治": "1642", "乐东黎族自治": "2032", "陵水黎族自治": "1643", "保亭黎族苗族自治": "1217", "琼中黎族苗族自治": "2031", "重庆": "132", "成都": "75", "自贡": "78", "攀枝花": "81", "泸州": "331", "德阳": "74", "绵阳": "240", "广元": "329", "遂宁": "330", "内江": "248", "乐山": "79", "南充": "291", "眉山": "77", "宜宾": "186", "广安": "241", "达州": "369", "雅安": "76", "巴中": "239", "资阳": "242", "阿坝藏族羌族自治州": "185", "甘孜藏族自治州": "73", "凉山彝族自治州": "80", "贵阳": "146", "六盘水": "147", "遵义": "262", "安顺": "263", "铜仁地区": "205", "黔西南布依族苗族自治州": "343", "毕节地区": "206", "黔东南苗族侗族自治州": "342", "黔南布依族苗族自治州": "306", "昆明": "104", "曲靖": "249", "玉溪": "106", "保山": "112", "昭通": "336", "丽江": "114", "临沧": "110", "楚雄彝族自治州": "105", "红河哈尼族彝族自治州": "107", "文山壮族苗族自治州": "177", "普洱": "108", "西双版纳傣族自治州": "109", "大理白族自治州": "111", "德宏傣族景颇族自治州": "116", "怒江傈僳族自治州": "113", "迪庆藏族自治州": "115", "拉萨": "100", "昌都地区": "99", "山南地区": "97", "日喀则地区": "102", "那曲地区": "101", "阿里地区": "103", "林芝地区": "98", "西安": "233", "铜川": "232", "宝鸡": "171", "咸阳": "323", "渭南": "170", "延安": "284", "汉中": "352", "榆林": "231", "安康": "324", "商洛": "285", "兰州": "36", "嘉峪关": "33", "金昌": "34", "白银": "35", "天水": "196", "武威": "118", "张掖": "117", "平凉": "359", "酒泉": "37", "庆阳": "135", "定西": "136", "陇南": "256", "临夏回族自治州": "182", "甘南藏族自治州": "247", "西宁": "66", "海东地区": "69", "海北藏族自治州": "67", "黄南藏族自治州": "70", "海南藏族自治州": "68", "果洛藏族自治州": "72", "玉树藏族自治州": "71", "海西蒙古族藏族自治州": "65", "银川": "360", "石嘴山": "335", "吴忠": "322", "固原": "246", "中卫": "181", "乌鲁木齐": "92", "克拉玛依": "95", "吐鲁番地区": "89", "哈密地区": "91", "昌吉回族自治州": "93", "博尔塔拉蒙古自治州": "88", "巴音郭楞蒙古自治州": "86", "阿克苏地区": "85", "克孜勒苏柯尔克孜自治州": "84", "喀什地区": "83", "和田地区": "82", "伊犁哈萨克自治州": "90", "塔城地区": "94", "阿勒泰地区": "96", "石河子": "770", "阿拉尔": "731", "图木舒克": "792", "五家渠": "789", "香港特别行政区": "2912", "澳门特别行政区": "2911"] diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 7dae269d1..36c0c1cae 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -1,24 +1,17 @@ package com.fangjia.script.gj.ccb -import com.fangjia.core.RegionData -import com.fangjia.core.util.DataUtil -import com.fangjia.core.util.DateUtil + import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.FangJiaAreaMapUtils import com.mongodb.BasicDBObject import com.mongodb.DBObject -import org.apache.commons.lang.StringUtils import org.bson.types.ObjectId import org.elasticsearch.index.query.BoolQueryBuilder import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource - -import java.sql.ResultSet -import java.text.SimpleDateFormat +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch import java.util.concurrent.ExecutorService @@ -26,12 +19,11 @@ import java.util.concurrent.Executors import java.util.concurrent.atomic.AtomicLong import java.util.regex.Matcher import java.util.regex.Pattern -import com.fangjia.script.util.FangJiaAreaMapUtils class HouseExport { - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy index 810f807b8..fd2856af7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/district/ExportData.groovy @@ -5,16 +5,14 @@ import com.fangjia.script.thirdPriceCrawl.CrawalDBUtil import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource -import java.sql.Connection import java.sql.ResultSet -import java.sql.Statement class ExportData { def static final DBCollection thirdDistrictColl = CrawalDBUtil.getThirdDistrictCol(); - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictWash.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictWash.groovy index b4a2ec125..b3001091a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictWash.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/districtwash/DistrictWash.groovy @@ -5,16 +5,11 @@ import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.script.commData.CommonDatautils import com.fangjia.script.gj.EsSearchUtils import com.fangjia.script.gj.district.FangjiaClientUtils -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.GeoHelper -import com.fangjia.script.util.GeoUtils +import com.fangjia.script.util.* import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils import com.mongodb.BasicDBObject import org.bson.types.ObjectId -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet import java.util.concurrent.CountDownLatch @@ -27,7 +22,7 @@ class DistrictWash { public static ExecutorService fixedThreadPool = Executors.newFixedThreadPool(5) static def totalCount = new AtomicLong() - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy index 20d3b1dff..81c1e3d8f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/CZBlankDataExport.groovy @@ -2,38 +2,28 @@ package com.fangjia.script.jiangnanblank import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DateUtil -import com.fangjia.script.util.ExcelUtils -import org.apache.commons.codec.digest.DigestUtils -import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource - -import java.sql.ResultSet -import java.text.DecimalFormat -import java.util.regex.Matcher -import java.util.regex.Pattern - -import net.sf.json.JSONObject - -import org.apache.poi.hssf.usermodel.HSSFWorkbook -import com.fangjia.datastore.driver.DSCollection; +import com.fangjia.datastore.driver.DSCollection import com.fangjia.datastore.driver.DataStore -import com.fangjia.script.commData.CommonDatautils; -import com.fangjia.script.util.DateUtils; -import com.fangjia.script.util.ExcelParser -import com.fangjia.script.common.DataHandleUtils; +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.common.DataHandleUtils +import com.fangjia.script.util.* import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject - -import org.apache.commons.io.FileUtils; -import org.bson.types.ObjectId - -import com.fangjia.script.util.MongoUtils; import com.mongodb.QueryOperators -import com.fangjia.script.util.OpenApiClient - -import java.util.Map.Entry; +import net.sf.json.JSONObject +import org.apache.commons.codec.digest.DigestUtils +import org.apache.commons.io.FileUtils +import org.apache.commons.lang.StringUtils +import org.apache.poi.hssf.usermodel.HSSFWorkbook +import org.bson.types.ObjectId +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.ResultSet +import java.text.DecimalFormat +import java.util.Map.Entry +import java.util.regex.Matcher +import java.util.regex.Pattern class CZBlankDataExport { @@ -62,7 +52,7 @@ class CZBlankDataExport { def directionMap = [1:"南",2:"北",3:"南北",4:"东",5:"西",6:"东西",7:"东南",8:"西南",9:"东北",10:"西北"] public static String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/JiangNanBlankHousesDataBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/JiangNanBlankHousesDataBatch.groovy index 9f955dcca..f66dc8ae1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/JiangNanBlankHousesDataBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jiangnanblank/JiangNanBlankHousesDataBatch.groovy @@ -12,7 +12,6 @@ import com.jcraft.jsch.ChannelSftp import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource /** * 江南银行线上数据交付 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/HousingOverviewExportBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/HousingOverviewExportBatch.groovy index c91e8e4f0..2f5e236d7 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/HousingOverviewExportBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/HousingOverviewExportBatch.groovy @@ -6,42 +6,27 @@ import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.crawl.CrawlCommandConsumerBatch import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ElasticUtils -import com.fangjia.script.util.ElasticsearchHelper -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBList import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DBCollection import com.mongodb.DBObject import org.apache.commons.lang.StringUtils -import org.elasticsearch.client.transport.TransportClient import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.BatchUpdateException import java.sql.ResultSet -import java.text.NumberFormat -import java.util.Map.Entry -import java.util.concurrent.BlockingQueue -import java.util.concurrent.ExecutorService -import java.util.concurrent.Executors import java.util.concurrent.LinkedBlockingDeque -import java.util.concurrent.LinkedBlockingQueue import java.util.concurrent.ThreadPoolExecutor import java.util.concurrent.TimeUnit -import java.util.regex.Pattern @BatchOwner(Watcher.WeiLei) class HousingOverviewExportBatch extends BaseBatch { static ThreadPoolExecutor _tpe = new ThreadPoolExecutor(JusfounCommon.CITIES.length, JusfounCommon.CITIES.length, 1, TimeUnit.HOURS, new LinkedBlockingDeque()) - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy index ae041304e..fbbd9b746 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/jusfoun/JusfounCommon.groovy @@ -1,6 +1,5 @@ package com.fangjia.script.jusfoun - import com.alibaba.fastjson.JSONObject import com.fangjia.core.util.DataUtil import com.fangjia.script.util.DateUtils @@ -18,9 +17,7 @@ import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.index.query.QueryBuilder import org.elasticsearch.index.query.QueryBuilders import org.elasticsearch.search.SearchHit -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource -import redis.clients.jedis.JedisPool -import redis.clients.jedis.JedisPoolConfig +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.* import java.text.DateFormat @@ -56,7 +53,7 @@ class JusfounCommon { public static Sql DB_Jusfoun_OLd = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun_backup?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver"); public static Sql DB_Jusfoun_New = Sql.newInstance("jdbc:mysql://192.168.0.21:3306/jusfoun?characterEncoding=utf-8&autoReconnect=true", "root", "Goojia123456", "com.mysql.jdbc.Driver"); - private static JdbcDataSource dataSource = null + private static EsDataSource dataSource = null private static Map _map_seaweed = new HashMap<>(); private static Map> mainCityMap = new HashMap<>(); @@ -390,7 +387,7 @@ class JusfounCommon { return DataUtil.hanyuQuanpin(hanzi, '') } - static JdbcDataSource getESDataSource() { + static EsDataSource getESDataSource() { if (dataSource != null) { return dataSource; } else { @@ -399,7 +396,7 @@ class JusfounCommon { return dataSource; } try { - dataSource = new JdbcDataSource(); + dataSource = new EsDataSource(); String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s"; dataSource.setUrl(address); Properties connectionProperties = new Properties(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/lixue/dataexport/AddressComExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/lixue/dataexport/AddressComExport.groovy index 56d0c3560..02ee077cd 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/lixue/dataexport/AddressComExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/lixue/dataexport/AddressComExport.groovy @@ -12,7 +12,7 @@ import org.apache.commons.lang.StringUtils import org.apache.commons.math.stat.descriptive.moment.StandardDeviation import org.apache.commons.math3.stat.descriptive.moment.Mean import org.bson.Document -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -22,7 +22,7 @@ import java.util.concurrent.ConcurrentHashMap import static com.mongodb.client.model.Filters.* def List find_rent_house_prices(String city, String region, String name, Date start, Date end){ - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogDisMonitorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogDisMonitorBatch.groovy index 9b29ae8db..db31c16bc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogDisMonitorBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogDisMonitorBatch.groovy @@ -1,20 +1,17 @@ package com.fangjia.script.log - -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource - -import java.sql.Connection -import java.sql.ResultSet -import java.sql.Statement - import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement @BatchOwner(Watcher.JiangYaHui) class MongoDBLogDisMonitorBatch extends BaseBatch { - JdbcDataSource dataSource = new JdbcDataSource(); + EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogMonitorBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogMonitorBatch.groovy index 9df954e22..eefeb9712 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogMonitorBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/log/MongoDBLogMonitorBatch.groovy @@ -1,20 +1,18 @@ package com.fangjia.script.log - -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource - -import java.sql.Connection -import java.sql.ResultSet -import java.sql.Statement - import com.fangjia.core.util.DateUtil import com.fangjia.script.BaseBatch import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement @BatchOwner(Watcher.JiangYaHui) class MongoDBLogMonitorBatch extends BaseBatch { - JdbcDataSource dataSource = new JdbcDataSource(); + EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.86:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; public final static def TOKEN="2307d2fec8314b3e8b4b83dd1d42da2e2051638e3e668f21e77ca01258bf682c" diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/rent/DistrictHouseTypeAnalyzeBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/rent/DistrictHouseTypeAnalyzeBatch.groovy index d46e0bddb..64d866e45 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/rent/DistrictHouseTypeAnalyzeBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/rent/DistrictHouseTypeAnalyzeBatch.groovy @@ -10,7 +10,8 @@ import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + import java.sql.ResultSet import java.util.concurrent.atomic.AtomicInteger @@ -25,7 +26,7 @@ class DistrictHouseTypeAnalyzeBatch extends BaseBatch { private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - private static final JdbcDataSource dataSource = new JdbcDataSource() + private static final EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy index cd7f1a29e..b5849e738 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/sync/AnalysisOpenUriLogSyncBatch.groovy @@ -53,8 +53,10 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch { void process(Object args) { Map userInfo = getUserInfo() - def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3)) - def endDate = DateUtil.truncateDay(new Date()) +// def startDate = DateUtil.truncateDay(DateUtil.add(new Date(), -3)) +// def endDate = DateUtil.truncateDay(new Date()) + def startDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210308")) + def endDate = DateUtil.truncateDay(Date.parse("yyyyMMdd", "20210314")) BasicDBObject query = new BasicDBObject() query.append("c_date", new BasicDBObject('$gte', startDate).append('$lt', endDate)) @@ -64,7 +66,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch { long count = 0 long total = cursor.count() print("total="+total) - BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("192.168.0.206", "fangjia") + BulkProcessor bulkProcessor = ElasticsearchHelper.createBulkProcessor("seawater.es.db.fangjia.com") for(BasicDBObject log : cursor){ Map parameter_map = (Map) log.get("parameter_map") if(parameter_map == null) { @@ -128,7 +130,7 @@ class AnalysisOpenUriLogSyncBatch extends BaseBatch { cmd.put("user_name", user.user_name) cmd.put("company_name", user.company) } - bulkProcessor.add(new IndexRequest("open_uri_log_normal", "_doc", log.get("_id").toString()).source(cmd)) + bulkProcessor.add(new IndexRequest("open_uri_log_normal").id(log.get("_id").toString()).source(cmd)) println(cmd) count++ if(count % 100 == 0){ diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy index 68933e6a1..3b78a5f46 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/ImportIndexStats.groovy @@ -1,50 +1,24 @@ package com.fangjia.script.trade -///导入价格指数 -import com.fangjia.script.BaseBatch; +import com.fangjia.core.util.DataUtil +import com.fangjia.script.BaseBatch + +///导入价格指数 import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher - -import org.apache.commons.io.FileUtils; - - - -import org.apache.commons.lang.StringUtils; -import org.apache.poi.xssf.usermodel.XSSFCell -import org.apache.poi.xssf.usermodel.XSSFWorkbook - +import com.fangjia.script.commData.CommonDatautils +import com.fangjia.script.util.DateUtils import com.fangjia.script.util.MongoUtils -import com.mongodb.QueryOperators -import com.fangjia.script.util.DateUtils; import com.mongodb.BasicDBObject -import com.mongodb.DBCollection; +import com.mongodb.DBCollection import com.mongodb.DBObject -import com.mongodb.Mongo - -import groovy.json.JsonSlurper -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.apache.commons.lang.StringUtils +import org.apache.commons.lang.math.RandomUtils +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet -import java.text.NumberFormat; -import java.util.regex.Matcher -import java.util.regex.Pattern -import com.fangjia.core.util.DBIterator - - - -import com.fangjia.core.util.DataUtil -import com.fangjia.core.util.StringUtil; -import com.fangjia.datastore.driver.DSCollection; -import com.fangjia.datastore.driver.DataStore -import com.fangjia.script.commData.CommonDatautils; -import com.ibm.icu.text.SimpleDateFormat; -import com.fangjia.storehouse.DBIterator -import com.fangjia.storehouse.DBIterator; -import com.fangjia.storehouse.StorehouseDB; - -import org.apache.commons.lang.math.RandomUtils; @BatchOwner(Watcher.GaoJuan) class ImportIndexStats extends BaseBatch{ //public final static DBCollection under_line = new Mongo("192.168.0.136", 27017).getDB('fangjia').getCollection('stats_index'); @@ -61,7 +35,7 @@ class ImportIndexStats extends BaseBatch{ public static final def dingjiYear = 2016; private static final String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - private static final JdbcDataSource dataSource = new JdbcDataSource() + private static final EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy index 36acf53e6..c68ba4a85 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/trade/TradeDealPriceStat.groovy @@ -1,17 +1,12 @@ package com.fangjia.script.trade import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.DateUtil import com.fangjia.script.util.DateUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.Bytes -import com.mongodb.DBCollection -import com.mongodb.DBObject -import com.mongodb.QueryOperators +import com.mongodb.* import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet @@ -22,7 +17,7 @@ class TradeDealPriceStat { public static String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() static { dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/AddressMatch.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/AddressMatch.groovy index f638b42f6..bede2bb9f 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/AddressMatch.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/AddressMatch.groovy @@ -3,7 +3,7 @@ package com.fangjia.script.usahouse import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet import java.util.concurrent.CountDownLatch @@ -14,7 +14,7 @@ import java.util.regex.Pattern class AddressMatch { - public static JdbcDataSource dataSource = new JdbcDataSource() + public static EsDataSource dataSource = new EsDataSource() private static final String address = "jdbc:es://http://192.168.0.228:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" static { dataSource.setUrl(address) diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/NewZealandWaiJuCrawl.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/NewZealandWaiJuCrawl.groovy index 5587d9a71..15b26e55c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/NewZealandWaiJuCrawl.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/usahouse/NewZealandWaiJuCrawl.groovy @@ -1,6 +1,6 @@ package com.fangjia.script.usahouse -import com.fangjia.core.util.BaiDuApiUtils + import com.fangjia.core.util.DataUtil import com.fangjia.httpclient.UserAgents import com.fangjia.script.BaseBatch @@ -8,14 +8,11 @@ import com.fangjia.script.BatchOwner import com.fangjia.script.Watcher import com.fangjia.script.crawl.JQueryUtils import com.fangjia.script.thirdPriceCrawl.CrawalUtil -import com.fangjia.script.util.HttpClient import com.fangjia.script.util.JacksonUtil import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import net.sf.json.JSON import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.select.Elements diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy index 4ddb4065d..7cbf5c37b 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/util/ElasticsearchHelper.groovy @@ -1,69 +1,125 @@ package com.fangjia.script.util -import org.apache.commons.lang3.SystemUtils + +import org.apache.http.HttpHost +import org.apache.http.auth.AuthScope +import org.apache.http.auth.UsernamePasswordCredentials +import org.apache.http.client.CredentialsProvider +import org.apache.http.client.config.RequestConfig +import org.apache.http.impl.client.BasicCredentialsProvider +import org.apache.http.impl.nio.client.HttpAsyncClientBuilder +import org.elasticsearch.action.ActionListener +import org.elasticsearch.action.bulk.BackoffPolicy import org.elasticsearch.action.bulk.BulkProcessor import org.elasticsearch.action.bulk.BulkRequest import org.elasticsearch.action.bulk.BulkResponse -import org.elasticsearch.client.transport.TransportClient -import org.elasticsearch.common.settings.Settings -import org.elasticsearch.common.transport.TransportAddress -import org.elasticsearch.xpack.client.PreBuiltXPackTransportClient +import org.elasticsearch.client.RestClient +import org.elasticsearch.client.RestClientBuilder +import org.elasticsearch.client.RestHighLevelClient +import org.elasticsearch.client.RequestOptions +import org.elasticsearch.common.unit.TimeValue +import org.elasticsearch.xpack.sql.jdbc.EsDataSource +import java.sql.Connection +import java.sql.ResultSet +import java.sql.Statement import java.util.concurrent.TimeUnit +import java.util.function.BiConsumer /** * 功能说明:Elasticsearch工具类 */ -class ElasticsearchHelper { +class ElasticsearchHelper { - static getClient(String host, String clusterName = "fangjia"){ - Settings.Builder builder = Settings.builder() - .put("cluster.name", clusterName) - .put("client.transport.sniff", false) - .put("client.transport.ping_timeout", "60s") - .put("xpack.security.user","developer:5e8edd851d2fdfbd7415232c67367cc3") - .put("xpack.security.transport.ssl.enabled", "true") - if(SystemUtils.IS_OS_WINDOWS){ - builder - .put("xpack.security.transport.ssl.truststore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12") - .put("xpack.security.transport.ssl.keystore.path", "D:\\goojiaconfig\\elasticmongo\\elastic-certificates.p12") - }else{ - builder - .put("xpack.security.transport.ssl.truststore.path", "/opt/elasticmongo/elastic-certificates.p12") - .put("xpack.security.transport.ssl.keystore.path", "/opt/elasticmongo/elastic-certificates.p12") + static getEsDataSource(String host){ + EsDataSource dataSource = new EsDataSource (); + dataSource.setUrl("jdbc:es://${host}:9200?timezone=UTC&page.size=240&connect.timeout=240&network.timeout=600&page.timeout=240"); + Properties connectionProperties = new Properties(); + connectionProperties.setProperty("user","developer"); + connectionProperties.setProperty("password","5e8edd851d2fdfbd7415232c67367cc3"); + dataSource.setProperties(connectionProperties); + return dataSource; + } + + static executeQuery(String sql, Closure closure, String host = "192.168.0.216"){ + EsDataSource dataSource = getEsDataSource(host) + Connection connection = dataSource.getConnection() + Statement statement = connection.createStatement() + ResultSet rs = statement.executeQuery(sql) + def columns = [] + for(int i = 1; i <= rs.getMetaData().getColumnCount(); i++){ + columns << rs.getMetaData().getColumnName(i) } - builder.put("xpack.security.transport.ssl.verification_mode", "certificate").build() - Settings settings = builder.build() - return new PreBuiltXPackTransportClient(settings) - .addTransportAddress(new TransportAddress(InetAddress.getByName(host), 9300)) + rs.getMetaData().getColumnCount() + while(rs.next()){ + def row = [:] + for (String col : columns){ + row[col] = rs.getObject(col) + } + closure.call(row) + } + rs.close() + connection.close() + } + + + static RestHighLevelClient getClient(String host) { + RestClientBuilder builder = RestClient.builder(new HttpHost(host, 9200, "http")); + builder.setRequestConfigCallback(new RestClientBuilder.RequestConfigCallback() { + @Override + RequestConfig.Builder customizeRequestConfig(RequestConfig.Builder requestConfigBuilder) { + requestConfigBuilder.setConnectionRequestTimeout(1000 * 60); + requestConfigBuilder.setSocketTimeout(1000 * 60); + return requestConfigBuilder; + } + }) + builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { + @Override + HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { + httpAsyncClientBuilder.disableAuthCaching(); + CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials("elastic", "goojia123456")); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + return httpAsyncClientBuilder; + } + }) + return new RestHighLevelClient(builder) } /** * 创建一个ES批处理器 - * @param clusterName * @param host * @return */ - static createBulkProcessor( String host, String clusterName){ - TransportClient client = getClient(host, clusterName) - BulkProcessor bulkProcessor = BulkProcessor.builder(client, new BulkProcessor.Listener() { + static createBulkProcessor(String host) { + BulkProcessor.Listener listener = new BulkProcessor.Listener() { @Override void beforeBulk(long executionId, BulkRequest request) { } @Override - void afterBulk(long executionId, BulkRequest request, BulkResponse response) { + void afterBulk(long executionId, BulkRequest request, + BulkResponse response) { } @Override - void afterBulk(long executionId, BulkRequest request, Throwable failure) { - failure.printStackTrace() + void afterBulk(long executionId, BulkRequest request, + Throwable failure) { } - }).setBulkActions(10000).setConcurrentRequests(20).build() + }; + RestHighLevelClient client = getClient(host); - return bulkProcessor + return BulkProcessor.builder(new BiConsumer>() { + @Override + void accept(BulkRequest bulkRequest, ActionListener bulkResponseActionListener) { + client.bulkAsync(bulkRequest, RequestOptions.DEFAULT, bulkResponseActionListener) + } + }, listener).setBulkActions(10000) + .setConcurrentRequests(20) + .setBackoffPolicy(BackoffPolicy.constantBackoff(TimeValue.timeValueSeconds(1L), 3)) + .setFlushInterval(TimeValue.timeValueSeconds(10L)).build(); } /** @@ -71,14 +127,13 @@ class ElasticsearchHelper { * @param * @return */ - static closeBulkProcessor(BulkProcessor bulkProcessor){ + static closeBulkProcessor(BulkProcessor bulkProcessor) { - bulkProcessor.awaitClose(10, TimeUnit.MINUTES) + bulkProcessor.awaitClose(10, TimeUnit.DAYS) bulkProcessor.flush() bulkProcessor.close() } - } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/weilei/Seabird_Seawater.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/weilei/Seabird_Seawater.groovy index b1b0e27be..d415e78c5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/weilei/Seabird_Seawater.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/weilei/Seabird_Seawater.groovy @@ -1,6 +1,6 @@ package com.fangjia.script.weilei -import com.fangjia.core.util.DateUtil + import com.fangjia.script.util.MongoUtils import com.mongodb.Bytes import com.mongodb.DBCollection @@ -8,7 +8,7 @@ import com.mongodb.DBCursor import com.mongodb.DBObject import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -16,7 +16,7 @@ import java.sql.SQLException import java.sql.Statement import java.text.SimpleDateFormat -JdbcDataSource dataSource = new JdbcDataSource(); +EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=120s&page.timeout=240s"; dataSource.setUrl(address); Properties connectionProperties = new Properties(); diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy index 4200cba90..d2ba9d116 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/Test3.groovy @@ -1,31 +1,16 @@ -import com.fangjia.core.Constants -import com.fangjia.core.RegionData import com.fangjia.core.util.Mongo3Utils -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.Jms -import com.fangjia.script.util.MongoUtils -import com.mongodb.BasicDBObject -import com.mongodb.CommandResult -import com.mongodb.DB -import com.mongodb.DBCollection -import com.mongodb.Mongo import com.mongodb.client.MongoCollection import org.bson.Document -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet import java.sql.Statement import java.util.function.Consumer -import static com.mongodb.client.model.Filters.*; - -import org.bson.Document -import com.mongodb.client.MongoCollection -import static com.mongodb.client.model.Filters.* -import com.fangjia.core.util.Mongo3Utils +import static com.mongodb.client.model.Filters.and +import static com.mongodb.client.model.Filters.eq def cityList = ["上海", "广州", "北京", "深圳", "成都", "杭州", "重庆", "武汉", "西安", "天津"] def roots = ["lianjia.com", "zhuge.com", "fang.com", "anjuke.com", "58.com", "soufy.cn"] @@ -43,7 +28,7 @@ cityList.each { def city -> } }) roots.each { def root -> - JdbcDataSource dataSource = new JdbcDataSource(); + EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/address/Test1.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/address/Test1.groovy index ef008f271..e67e057e5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/address/Test1.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/address/Test1.groovy @@ -1,23 +1,12 @@ package com.fangjia.script.wenyan.address -import com.fangjia.core.util.BaiDuApiUtils -import com.fangjia.core.util.DateUtil -import com.fangjia.core.util.HttpRequest + import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient -import com.fangjia.script.yinjihaun.util.QQKeyPool import com.mongodb.BasicDBObject -import com.mongodb.DBCollection -import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -import org.apache.lucene.index.Term -import org.apache.lucene.search.BooleanClause -import org.apache.lucene.search.BooleanQuery -import org.apache.lucene.search.TermQuery import org.apache.solr.client.solrj.util.ClientUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -44,7 +33,7 @@ import java.sql.Statement // data << (k.split("_") as List) //} //ExcelUtils.write("D:\\无均价数据.xlsx", data, "结果") -JdbcDataSource dataSource = new JdbcDataSource(); +EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.18:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s"; dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/avgPrice/Test4.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/avgPrice/Test4.groovy index e4ef2b30d..91648461d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/avgPrice/Test4.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/avgPrice/Test4.groovy @@ -1,20 +1,20 @@ package com.fangjia.script.wenyan.avgPrice - import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import org.apache.commons.lang.StringUtils import org.apache.solr.client.solrj.util.ClientUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource + import java.sql.Connection import java.sql.ResultSet import java.sql.Statement def findEstateType(String city, String region, String name, List alias) { - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/rent/RentStat.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/rent/RentStat.groovy index 80e188b26..9d56a3720 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/rent/RentStat.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/rent/RentStat.groovy @@ -10,17 +10,12 @@ import org.apache.commons.lang.StringUtils import org.apache.commons.math3.ml.clustering.CentroidCluster import org.apache.commons.math3.ml.clustering.DoublePoint import org.apache.commons.math3.ml.clustering.KMeansPlusPlusClusterer -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet import java.sql.Statement -import org.bson.Document -import com.mongodb.client.MongoCollection -import static com.mongodb.client.model.Filters.* -import com.fangjia.core.util.Mongo3Utils - DBCollection RENT_FACTOR_MAP = MongoUtils.getCollectionByName("fangjia", "rent_factor_map", "192.168.0.136", 28017) def findHouse(BasicDBObject district){ @@ -54,7 +49,7 @@ def findHouse(BasicDBObject district){ sqlBuffer.append(" and root !='focus.cn' and root != '58.com' ") sqlBuffer.append(" ORDER BY p_date desc") - JdbcDataSource dataSource = new JdbcDataSource(); + EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://seawater.es.db.fangjia.com/timezone=UTC&page.size=240&connect.timeout=240s&network.timeout=600s&page.timeout=240s"; dataSource.setUrl(address); Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterDirection.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterDirection.groovy index 9140a689a..436395151 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterDirection.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterDirection.groovy @@ -1,13 +1,13 @@ package com.fangjia.script.wenyan.seawater import com.fangjia.script.util.ExcelUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet import java.sql.Statement -JdbcDataSource dataSource = new JdbcDataSource(); +EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterFitment.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterFitment.groovy index 13b6a1341..4a75f9886 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterFitment.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/seawater/StatsSeawaterFitment.groovy @@ -1,13 +1,13 @@ package com.fangjia.script.wenyan.seawater import com.fangjia.script.util.ExcelUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet import java.sql.Statement -JdbcDataSource dataSource = new JdbcDataSource(); +EsDataSource dataSource = new EsDataSource(); String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ChangZou.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ChangZou.groovy index 213d46ff9..a4145041c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ChangZou.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ChangZou.groovy @@ -1,20 +1,12 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.util.HttpRequest + import com.fangjia.core.util.Mongo3Utils -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ExcelUtils -import com.mongodb.BasicDBObject import com.mongodb.client.MongoCollection import org.bson.Document -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource -import java.sql.Connection -import java.sql.ResultSet -import java.sql.Statement -import java.util.function.Consumer -import static com.mongodb.client.model.Filters.and import static com.mongodb.client.model.Filters.eq + // //def data = [["城市", "区域", "小区名", "2018", "全部"]] //cityList = ["上海"] @@ -47,10 +39,6 @@ import static com.mongodb.client.model.Filters.eq //HttpRequest httpRequest = new HttpRequest() //println httpRequest.get("https://sh.centanet.com/xiaoqu/pudongxinqu/g2", [:], ['User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/73.0.3683.86 Safari/537.36'], "UTF-8") -import org.bson.Document -import com.mongodb.client.MongoCollection -import static com.mongodb.client.model.Filters.* -import com.fangjia.core.util.Mongo3Utils MongoCollection AVERAGER_CONFIG = Mongo3Utils.getDbCollection("fangjia", "averager_config") AVERAGER_CONFIG.find().each {Document doc-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy index 525432fe4..3b51386dc 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/QinLin.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20201101", "yyyyMMdd") -def endDate = DateUtil.parseDate("20201201", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210301", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210401", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SanSha.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SanSha.groovy index 33aaa487a..90e849c7c 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SanSha.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SanSha.groovy @@ -1,19 +1,13 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.RegionData -import com.fangjia.core.util.DateUtil -import com.fangjia.core.util.HttpRequest -import com.fangjia.core.util.Mongo3Utils + import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import com.mongodb.Mongo -import com.mongodb.client.MongoCollection import org.apache.commons.lang.StringUtils -import org.bson.Document -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -61,7 +55,7 @@ import java.sql.Statement //println(bigMarkupCount) def countHouses(String city, String region, String name){ - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Seaweed.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Seaweed.groovy index 825938c8b..b199f2ca5 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Seaweed.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Seaweed.groovy @@ -1,28 +1,16 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.Constants -import com.fangjia.core.util.DateUtil -import com.fangjia.core.util.HttpRequest -import com.fangjia.datastore.driver.DSCollection -import com.fangjia.datastore.driver.DataStore + import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.OpenApiClient import com.mongodb.BasicDBObject -import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -import org.bson.Document -import com.mongodb.client.MongoCollection -import org.bson.types.ObjectId -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet import java.sql.Statement -import java.util.function.Consumer -import static com.mongodb.client.model.Filters.* -import com.fangjia.core.util.Mongo3Utils def poiData = Collections.synchronizedList([]) // //ExcelUtils.multiThreadingModify("D:\\商务数据\\附件1:测试数据\\房价网测试.xlsx", { List item -> @@ -79,7 +67,7 @@ def poiData = Collections.synchronizedList([]) // println(item) //},1) def findHouseCount(String city, String region, String name, List alias) { - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy index c75c8611b..c03f780d8 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -11,8 +11,8 @@ import org.bson.Document import static com.mongodb.client.model.Filters.* MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") -def startDate = DateUtil.parseDate("20210201", "yyyyMMdd") -def endDate = DateUtil.parseDate("20210301", "yyyyMMdd") +def startDate = DateUtil.parseDate("20210301", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210401", "yyyyMMdd") Calendar calendar = Calendar.getInstance() calendar.setTime(startDate) def count = 0 diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy index b31c40a59..8c05b9c5a 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test.groovy @@ -1,11 +1,17 @@ import com.fangjia.core.RegionData +import com.fangjia.core.util.BaiDuApiUtils import com.fangjia.core.util.DataUtil import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.GeoUtils +import com.fangjia.core.util.GeocodeUtils import com.fangjia.core.util.HttpRequest import com.fangjia.datastore.driver.DSCursor import com.fangjia.datastore.driver.FacetResult +import com.fangjia.script.util.BaiduMapUtils import com.fangjia.script.util.CommonUtils import com.fangjia.script.util.DbUtils +import com.fangjia.script.util.ExcelUtils + //import com.fangjia.script.util.ExcelUtils import com.fangjia.script.util.MongoUtils import com.fangjia.script.util.OpenApiClient @@ -132,4 +138,70 @@ import org.apache.commons.lang3.StringUtils // item["E"] = RegionData.formatRegion(item["B"], item["C"]) // println(item) //}) -CommonUtils.reFitAverage("常州","新北","龙城花园", "普通住宅") \ No newline at end of file +//def border = [13511506.7,3639168.61,13511516.8,3639404.86,13511515.27,3639416.92,13511738.21,3639384.94,13512007.72,3639336.35,13511984.92,3639074.96,13511722.75,3639121.08,13511506.7,3639168.61] +//def coords = [] +//for(int i=0; i < border.size() - 1; i=i+2){ +// coords << "${border[i]},${border[i+1]}" +//} +//def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5) +//println(json) +//def points = [] +//for(JSONObject it : json.result){ +// def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y) +// points << "${geo.lng},${geo.lat}" +// println(" new AMap.LngLat(${geo.lng},${geo.lat}),") +//} + +DBCollection BAIDUMAP_BORDER_COLL = MongoUtils.getCollectionByName("addrepo", "baidumap_border", "192.168.0.206", 27020) +BAIDUMAP_BORDER_COLL.find(new BasicDBObject("baidu_border", null)).each {BasicDBObject dbo-> + def border = dbo.getString("border").replace("1-", "").replace(";", "").split(",") as List + def coords = [] + def baidu_points = [] + def gaode_points = [] + for(int i=0; i < border.size() - 1; i=i+2){ + coords << "${border[i]},${border[i+1]}" + if(coords.size() > 50){ + def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5) + for(JSONObject it : json.result){ + baidu_points << "${it.x},${it.y}" + def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y) + gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}" + } + coords.clear() + } + } + if(coords.size() > 0){ + def json=BaiDuApiUtils.geoconv(coords.join(";"), 6, 5) + for(JSONObject it : json.result){ + baidu_points << "${it.x},${it.y}" + def geo = GeocodeUtils.geoconvBaiduToGoogle(it.x, it.y) + gaode_points << "${DataUtil.roundDouble(geo.lng, 6)},${DataUtil.roundDouble(geo.lat,6)}" + } + } + dbo.format_city = RegionData.formatCity(dbo.city) + dbo.baidu_border = baidu_points.join(";") + dbo.gaode_border = gaode_points.join(";") + BAIDUMAP_BORDER_COLL.save(dbo) + println(dbo) +} + +//ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\人口测试数据.xlsx",{List item-> +// def seaweed = DbUtils.seaweed.findOne(DbUtils.buildSeaweedQuery(item["B"], item["C"], item["D"]), new BasicDBObject("alias", 1).append("lng",1).append("lat", 1)) +// if(seaweed){ +// item["A"] = seaweed._id.toString() +// def names = [item['D']] +// if(seaweed.alias){ +// names.addAll(seaweed.alias) +// } +// names.unique() +// if(seaweed.lng){ +// item["G"] = DataUtil.roundDouble(seaweed.lng/10000000000d, 6) +// item["H"] = DataUtil.roundDouble(seaweed.lat/10000000000d, 6) +// } +// def one = BAIDUMAP_BORDER_COLL.findOne(new BasicDBObject("format_city", item["B"]).append("format_region", item["C"]).append("name", new BasicDBObject('$in', names))) +// if(one){ +// item["I"] = one.baidu_border +// } +// println(item) +// } +//}) \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test11.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test11.groovy index 52e09b50f..20e49c20d 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test11.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test11.groovy @@ -669,7 +669,6 @@ def findRentInfoByRoom(String city, String region){ def findRent(def city, def region, def name){ - DBCollection DISTRICT_STAT_RENT = MongoUtils.getCollectionByName("fangjia", "sqmetre_rent", "mongo.db.fangjia.com", 27017) def query = new BasicDBObject() query.city = city query.region = region diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test16.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test16.groovy index 31493e583..31f809249 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test16.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test16.groovy @@ -1,19 +1,8 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.util.DateUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.yinjihaun.amap.GaoDeApiUtils -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection -import net.sf.json.JSONArray -import net.sf.json.JSONObject -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource -import java.sql.Connection -import java.sql.ResultSet -import java.sql.Statement +import com.fangjia.script.util.ExcelUtils +import com.mongodb.BasicDBObject //DBCollection AVERAGE_STATISTICS = MongoUtils.getCollectionByName("fangjia", "average_statistics", "mongo.db.fangjia.com") ////ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Downloads\\下载记录20200818-结果 - 副本.xlsx",{List item-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test20.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test20.groovy index 03cb1c73f..a30df93f3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test20.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test20.groovy @@ -1,23 +1,9 @@ package com.fangjia.script.wenyan.temp -import com.fangjia.core.RegionData -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.DataUtil -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ExcelUtils -import com.fangjia.script.util.FangJiaAreaMapUtils -import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient -import com.fangjia.script.util.ProvinceData -import com.mongodb.BasicDBObject -import com.mongodb.DBCollection -import net.sf.json.JSONObject -import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource -import java.sql.Connection -import java.sql.ResultSet -import java.sql.Statement +import com.fangjia.script.util.ExcelUtils +import com.fangjia.script.util.MongoUtils +import com.mongodb.DBCollection // //ExcelUtils.multiThreadingModify("D:\\数据\\建行学校数据\\上海学校_0811.xls",{List item-> diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test8.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test8.groovy index 854941ecd..2d73603c3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test8.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/Test8.groovy @@ -5,13 +5,12 @@ import com.fangjia.script.util.DbUtils import com.fangjia.script.util.ExcelUtils import com.mongodb.BasicDBObject import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet import java.sql.Statement - //def data = [] //data << ["城市", "区域", "小区名", "近一月", "近三月", "近6月", "近一年", "户数"] //DbUtils.seaweed.find(DbUtils.buildSeaweedQuery("上海")).each {BasicDBObject district-> @@ -36,7 +35,7 @@ import java.sql.Statement //ExcelUtils.write("D:\\上海成交统计.xlsx", data, "成交数据") def find_rent_house_prices(String city, String region, String name){ - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() @@ -80,7 +79,7 @@ def find_rent_house_prices(String city, String region, String name){ } ExcelUtils.multiThreadingModify("D:\\上海成交统计.xlsx", {List item-> - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.45:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/XingYe.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/XingYe.groovy index 3f714fd48..2ebfa4262 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/XingYe.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/XingYe.groovy @@ -1,13 +1,12 @@ package com.fangjia.script.wenyan.temp - import com.fangjia.script.util.* import com.mongodb.BasicDBObject import com.mongodb.DBCollection import com.mongodb.DBObject import net.sf.json.JSONObject import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.Connection import java.sql.ResultSet @@ -16,7 +15,7 @@ import java.sql.Statement DBCollection DIST_COLL = MongoUtils.getCollectionByName("fangjia_stat", "district_grade", "192.168.0.43") def String hasElevator(BasicDBObject seaweed){ - JdbcDataSource dataSource = new JdbcDataSource() + EsDataSource dataSource = new EsDataSource() String address = "jdbc:es://http://192.168.0.206:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" dataSource.setUrl(address) Properties connectionProperties = new Properties() diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy index 4c399a4e5..eb226eba3 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/xiaoqugeo.groovy @@ -1,40 +1,7 @@ package com.fangjia.script.wenyan.temp +import com.fangjia.script.util.ElasticsearchHelper -import com.fangjia.core.util.DateUtil -import com.fangjia.script.util.CommonUtils -import com.fangjia.script.util.DbUtils -import com.fangjia.script.util.ExcelUtils -import com.mongodb.BasicDBObject - -ExcelUtils.read("D:\\Users\\wenyan\\Desktop\\新增小区.xlsx", 0, 0).each {List item-> - - def keys = new BasicDBObject() - keys.city = 1 - keys.region = 1 - keys.name = 1 - keys.estate_type2 = 1 - def query = DbUtils.buildSeaweedQuery(item["A"], item["B"], item["C"]) - query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1))) - DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo-> - if(!DbUtils.findDistrictStat(dbo, new Date())){ - CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2) - println(dbo) - } -// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2) -// println(dbo) - } -} -//["佛山","济南","北京","广州","深圳"].each {def city-> -// def keys = new BasicDBObject() -// keys.city = 1 -// keys.region = 1 -// keys.name = 1 -// keys.estate_type2 = 1 -// def query = DbUtils.buildSeaweedQuery(city) -// query.put('c_date', new BasicDBObject('$gt', DateUtil.add(DateUtil.truncateDay(new Date()), -1))) -// DbUtils.seaweed.find(query, keys).each {BasicDBObject dbo-> -// CommonUtils.recalculationAverage(dbo.city, dbo.region, dbo.name, dbo.estate_type2) -// println(dbo) -// } -//} +ElasticsearchHelper.executeQuery("select * from seawater where p_date > 20210407 limit 10",{Map row-> + println(row.name) +}, "192.168.0.216") \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/price/SaveDistrict.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/price/SaveDistrict.groovy index 9965c9b02..c6226b6b0 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/price/SaveDistrict.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/yinjihaun/price/SaveDistrict.groovy @@ -1,20 +1,14 @@ package com.fangjia.script.yinjihaun.price -import com.fangjia.core.util.HttpRequest -import com.fangjia.script.commData.CommonDatautils -import com.fangjia.script.util.DateUtils -import com.fangjia.script.util.ExcelUtils + import com.fangjia.script.util.MongoUtils -import com.fangjia.script.util.OpenApiClient -import com.mongodb.BasicDBObject import com.mongodb.DBCollection -import org.apache.commons.lang.StringUtils -import org.elasticsearch.xpack.sql.jdbc.jdbcx.JdbcDataSource +import org.elasticsearch.xpack.sql.jdbc.EsDataSource import java.sql.ResultSet String address = "jdbc:es://http://seawater.es.db.fangjia.com:9200/timezone=UTC&page.size=250&connect.timeout=120s&network.timeout=120s&page.timeout=120s" -JdbcDataSource dataSource = new JdbcDataSource() +EsDataSource dataSource = new EsDataSource() dataSource.setUrl(address) Properties connectionProperties = new Properties() connectionProperties.put("user", "developer")