This commit is contained in:
2021-03-16 11:58:46 +08:00
parent 3764b2876b
commit 40013fb01d
3 changed files with 84 additions and 11 deletions
+5
View File
@@ -115,6 +115,11 @@
<artifactId>fangjia-datastore-driver</artifactId>
<version>3.9.1</version>
</dependency>
<dependency>
<groupId>cc.blynk.clickhouse</groupId>
<artifactId>clickhouse4j</artifactId>
<version>1.4.4</version>
</dependency>
<dependency>
<groupId>redis.clients</groupId>
<artifactId>jedis</artifactId>
@@ -0,0 +1,69 @@
package com.fangjia.script.wenyan.temp
import cc.blynk.clickhouse.copy.CopyManager
import cc.blynk.clickhouse.copy.CopyManagerFactory
import com.fangjia.core.util.DateUtil
import net.sf.json.JSONNull
import net.sf.json.JSONObject
import ru.yandex.clickhouse.ClickHouseConnection
import ru.yandex.clickhouse.ClickHouseDriver
import ru.yandex.clickhouse.settings.ClickHouseProperties
import java.sql.PreparedStatement
import java.sql.Timestamp
ClickHouseDriver clickHouseDriver = new ClickHouseDriver()
ClickHouseProperties clickHouseProperties = new ClickHouseProperties()
clickHouseProperties.connectionTimeout = 1000 * 3600 * 24
clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24
clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24
clickHouseProperties.user = "default"
clickHouseProperties.password = "goojia123456"
ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties)
CopyManager copyManager = CopyManagerFactory.create(connection)
copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv");
//PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
//def cols = ["id", "city", "region", "name", "district_name", "address", "room", "hall", "toilet", "area", "avg_price", "total_price ", "floor_info", "height", "floor", "direction", "fitment", "p_date", "publish_date", "c_date", "m_date"]
//int count = 0
//new File("D:\\½¨ÐвâÊÔÊý¾Ý").eachFile { def file ->
// file.eachLine("UTF-8", { def line ->
// String id = line.substring(0, line.indexOf("\t"))
// String jsonStr = line.substring(line.indexOf("\t") + 1)
// count++
// JSONObject json = JSONObject.fromObject(jsonStr)
// json.put("id", id)
// for (int i = 0; i < cols.size(); i++) {
// if (cols[i].toString().endsWith("date") && "p_date" != cols[i]) {
// if ("c_date" == cols[i]) {
// if (json.get("m_date") instanceof Long) {
// sth.setTimestamp(i + 1, new Timestamp(json.get("m_date")))
// } else {
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get("m_date"))).getTime()))
//
// }
// } else {
// if (json.get(cols[i]) instanceof String) {
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get(cols[i]))).getTime()))
// } else {
// sth.setTimestamp(i + 1, new Timestamp(json.get(cols[i])))
//
// }
// }
// } else {
// if (json.get(cols[i]) instanceof JSONNull) {
// sth.setObject(i + 1, null)
// } else {
// sth.setObject(i + 1, json.get(cols[i]))
// }
// }
// }
// sth.addBatch()
// if (count % 30000 == 0) {
// sth.executeBatch()
// println(new Date().format("HH:mm:ss " + count))
// }
// })
//}
//sth.execute()
@@ -1,17 +1,16 @@
package com.fangjia.script.wenyan.temp
import com.fangjia.core.ProvinceData
import com.fangjia.core.RegionData
import com.fangjia.script.util.CommonUtils
import com.fangjia.script.util.DbUtils
import com.fangjia.script.util.ExcelUtils
import com.mongodb.BasicDBObject
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\½¨ÐÐÉÊý¾Ýͳ¼Æ.xlsx",{List item->
String city = RegionData.formatCity(item["A"])
if(city){
item["G"] = DbUtils.seaweed.count(DbUtils.buildSeaweedQuery(RegionData.formatCity(item["A"])))
}else{
item["G"] = null
def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]]
["ID", "北京", "重庆", "哈尔滨"].each {def city->
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district->
def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")]
println(item)
data << item
}
})
}
ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据")