From 213330cfdda120db29a0c18680b9b11c047e3654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=8E=8B=E6=96=87=E8=89=B3?= Date: Thu, 25 Mar 2021 10:38:41 +0800 Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E5=9B=9B=E5=B7=9D=E9=94=A6?= =?UTF-8?q?=E7=A8=8B=E4=BB=A3=E7=A0=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../fangjia/script/gj/ccb/HouseExport.groovy | 6 +-- .../script/wenyan/temp/ClickHouseDemo.groovy | 35 +++++++++++----- .../wenyan/temp/SiChuanJingCheng.groovy | 42 +++++++++++++++++++ 3 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy index 3c4ca8b31..84bf1edc9 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/gj/ccb/HouseExport.groovy @@ -78,11 +78,11 @@ class HouseExport { // exportRentCSV(city, districts, 20200201, 20210301, 2020); }else{ - exportRentCSV(city, districts, 20170801, 20181001, 2017) - exportRentCSV(city, districts, 20200201, 20210301, 2020); +// exportRentCSV(city, districts, 20170801, 20181001, 2017) +// exportRentCSV(city, districts, 20200201, 20210301, 2020); exportSellCSV(city, districts, 20170801, 20181001, 2017); - exportSellCSV(city, districts, 20200201, 20210301, 2020); +// exportSellCSV(city, districts, 20200201, 20210301, 2020); } diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy index 71df8ece9..5eed415f1 100644 --- a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/ClickHouseDemo.groovy @@ -5,6 +5,7 @@ import cc.blynk.clickhouse.copy.CopyManagerFactory import com.fangjia.core.util.DateUtil import net.sf.json.JSONNull import net.sf.json.JSONObject +import org.bson.types.ObjectId import ru.yandex.clickhouse.ClickHouseConnection import ru.yandex.clickhouse.ClickHouseDriver import ru.yandex.clickhouse.settings.ClickHouseProperties @@ -12,16 +13,16 @@ import ru.yandex.clickhouse.settings.ClickHouseProperties import java.sql.PreparedStatement import java.sql.Timestamp -ClickHouseDriver clickHouseDriver = new ClickHouseDriver() -ClickHouseProperties clickHouseProperties = new ClickHouseProperties() -clickHouseProperties.connectionTimeout = 1000 * 3600 * 24 -clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24 -clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24 -clickHouseProperties.user = "default" -clickHouseProperties.password = "goojia123456" -ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties) -CopyManager copyManager = CopyManagerFactory.create(connection) -copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv"); +//ClickHouseDriver clickHouseDriver = new ClickHouseDriver() +//ClickHouseProperties clickHouseProperties = new ClickHouseProperties() +//clickHouseProperties.connectionTimeout = 1000 * 3600 * 24 +//clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24 +//clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24 +//clickHouseProperties.user = "default" +//clickHouseProperties.password = "goojia123456" +//ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties) +//CopyManager copyManager = CopyManagerFactory.create(connection) +//copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv"); //PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)") @@ -66,4 +67,16 @@ copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv"); // } // }) //} -//sth.execute() \ No newline at end of file +//sth.execute() +def ids = new HashSet() +for(int i =0 ; i < 10000 * 10000; i++){ + if(i%10000 == 0){ + println(i) + } + def id = new ObjectId().toString() + if(ids.contains(id)){ + println(id) + }else{ + ids << id + } +} \ No newline at end of file diff --git a/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy new file mode 100644 index 000000000..c75c8611b --- /dev/null +++ b/projects/fangjia-script/src/main/groovy/com/fangjia/script/wenyan/temp/SiChuanJingCheng.groovy @@ -0,0 +1,42 @@ +package com.fangjia.script.wenyan.temp + +import com.fangjia.core.Constants +import com.fangjia.core.util.DateUtil +import com.fangjia.core.util.Mongo3Utils +import com.fangjia.script.util.ExcelUtils +import com.mongodb.client.MongoCollection +import net.sf.json.JSONObject +import org.bson.Document + +import static com.mongodb.client.model.Filters.* + +MongoCollection OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log") +def startDate = DateUtil.parseDate("20210201", "yyyyMMdd") +def endDate = DateUtil.parseDate("20210301", "yyyyMMdd") +Calendar calendar = Calendar.getInstance() +calendar.setTime(startDate) +def count = 0 +def hashSet = [] +def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]] +while (startDate.before(endDate)) { + calendar.add(Calendar.DATE, 1) + def nextDate = calendar.getTime() + count++ + OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc -> + def result = JSONObject.fromObject(doc.res_map.result) + if(!result.buildYear || !result.avgPrice){ + return + } + def params = JSONObject.fromObject(doc.parameter_map) + def key = "${params.address}_${params.size}".toString() + if (!hashSet.contains(key)) { + def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()] + data << item + hashSet << key + } else { + println("已存在") + } + } + startDate = nextDate +} +ExcelUtils.write("D:\\四川锦城${startDate.format('yyyyMM')}.xlsx", data, "查询日志") \ No newline at end of file