添加四川锦程代码

This commit is contained in:
2021-03-25 10:38:41 +08:00
parent f8226b1f69
commit 213330cfdd
3 changed files with 69 additions and 14 deletions
@@ -78,11 +78,11 @@ class HouseExport {
// exportRentCSV(city, districts, 20200201, 20210301, 2020);
}else{
exportRentCSV(city, districts, 20170801, 20181001, 2017)
exportRentCSV(city, districts, 20200201, 20210301, 2020);
// exportRentCSV(city, districts, 20170801, 20181001, 2017)
// exportRentCSV(city, districts, 20200201, 20210301, 2020);
exportSellCSV(city, districts, 20170801, 20181001, 2017);
exportSellCSV(city, districts, 20200201, 20210301, 2020);
// exportSellCSV(city, districts, 20200201, 20210301, 2020);
}
@@ -5,6 +5,7 @@ import cc.blynk.clickhouse.copy.CopyManagerFactory
import com.fangjia.core.util.DateUtil
import net.sf.json.JSONNull
import net.sf.json.JSONObject
import org.bson.types.ObjectId
import ru.yandex.clickhouse.ClickHouseConnection
import ru.yandex.clickhouse.ClickHouseDriver
import ru.yandex.clickhouse.settings.ClickHouseProperties
@@ -12,16 +13,16 @@ import ru.yandex.clickhouse.settings.ClickHouseProperties
import java.sql.PreparedStatement
import java.sql.Timestamp
ClickHouseDriver clickHouseDriver = new ClickHouseDriver()
ClickHouseProperties clickHouseProperties = new ClickHouseProperties()
clickHouseProperties.connectionTimeout = 1000 * 3600 * 24
clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24
clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24
clickHouseProperties.user = "default"
clickHouseProperties.password = "goojia123456"
ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties)
CopyManager copyManager = CopyManagerFactory.create(connection)
copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv");
//ClickHouseDriver clickHouseDriver = new ClickHouseDriver()
//ClickHouseProperties clickHouseProperties = new ClickHouseProperties()
//clickHouseProperties.connectionTimeout = 1000 * 3600 * 24
//clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24
//clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24
//clickHouseProperties.user = "default"
//clickHouseProperties.password = "goojia123456"
//ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties)
//CopyManager copyManager = CopyManagerFactory.create(connection)
//copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv");
//PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
@@ -66,4 +67,16 @@ copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv");
// }
// })
//}
//sth.execute()
//sth.execute()
def ids = new HashSet()
for(int i =0 ; i < 10000 * 10000; i++){
if(i%10000 == 0){
println(i)
}
def id = new ObjectId().toString()
if(ids.contains(id)){
println(id)
}else{
ids << id
}
}
@@ -0,0 +1,42 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.Constants
import com.fangjia.core.util.DateUtil
import com.fangjia.core.util.Mongo3Utils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.client.MongoCollection
import net.sf.json.JSONObject
import org.bson.Document
import static com.mongodb.client.model.Filters.*
MongoCollection<Document> OPEN_URI_LOG = Mongo3Utils.getDbCollection(Constants.SEAWATER_MONGO_URI, "business", "open_uri_log")
def startDate = DateUtil.parseDate("20210201", "yyyyMMdd")
def endDate = DateUtil.parseDate("20210301", "yyyyMMdd")
Calendar calendar = Calendar.getInstance()
calendar.setTime(startDate)
def count = 0
def hashSet = []
def data = [["用户ID", "用户名称", "IP", "访问链接", "访问时间", "dataStatus", "code", "msg", "参数", "resMap"]]
while (startDate.before(endDate)) {
calendar.add(Calendar.DATE, 1)
def nextDate = calendar.getTime()
count++
OPEN_URI_LOG.find(and(eq("user_id", "1007721"), eq("request_uri", "/property/evaluate3"), eq("data_status", 1), lt("c_date", nextDate), gte("c_date", startDate))).each { Document doc ->
def result = JSONObject.fromObject(doc.res_map.result)
if(!result.buildYear || !result.avgPrice){
return
}
def params = JSONObject.fromObject(doc.parameter_map)
def key = "${params.address}_${params.size}".toString()
if (!hashSet.contains(key)) {
def item = ["1007721", "scjc", doc.ip, doc.request_uri, doc.c_date, doc.data_status, doc.code, doc.msg, JSONObject.fromObject(doc.parameter_map).toString(), result.toString()]
data << item
hashSet << key
} else {
println("已存在")
}
}
startDate = nextDate
}
ExcelUtils.write("D:\\四川锦城${startDate.format('yyyyMM')}.xlsx", data, "查询日志")