update
This commit is contained in:
@@ -115,6 +115,11 @@
|
||||
<artifactId>fangjia-datastore-driver</artifactId>
|
||||
<version>3.9.1</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>cc.blynk.clickhouse</groupId>
|
||||
<artifactId>clickhouse4j</artifactId>
|
||||
<version>1.4.4</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>redis.clients</groupId>
|
||||
<artifactId>jedis</artifactId>
|
||||
|
||||
+69
@@ -0,0 +1,69 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import cc.blynk.clickhouse.copy.CopyManager
|
||||
import cc.blynk.clickhouse.copy.CopyManagerFactory
|
||||
import com.fangjia.core.util.DateUtil
|
||||
import net.sf.json.JSONNull
|
||||
import net.sf.json.JSONObject
|
||||
import ru.yandex.clickhouse.ClickHouseConnection
|
||||
import ru.yandex.clickhouse.ClickHouseDriver
|
||||
import ru.yandex.clickhouse.settings.ClickHouseProperties
|
||||
|
||||
import java.sql.PreparedStatement
|
||||
import java.sql.Timestamp
|
||||
|
||||
ClickHouseDriver clickHouseDriver = new ClickHouseDriver()
|
||||
ClickHouseProperties clickHouseProperties = new ClickHouseProperties()
|
||||
clickHouseProperties.connectionTimeout = 1000 * 3600 * 24
|
||||
clickHouseProperties.keepAliveTimeout = 1000 * 3600 * 24
|
||||
clickHouseProperties.dataTransferTimeout = 1000 * 3600 * 24
|
||||
clickHouseProperties.user = "default"
|
||||
clickHouseProperties.password = "goojia123456"
|
||||
ClickHouseConnection connection = clickHouseDriver.connect("jdbc:clickhouse://192.168.0.93:8123/fangjia", clickHouseProperties)
|
||||
CopyManager copyManager = CopyManagerFactory.create(connection)
|
||||
copyManager.copyFromDb("select * from fangjia.SEAWATER", "D://123.csv");
|
||||
|
||||
|
||||
//PreparedStatement sth = connection.prepareStatement("INSERT INTO SEAWATER(id,city,region,name,district_name,address,room,hall,toilet,area,avg_price,total_price ,floor_info,height,floor,direction,fitment,p_date,publish_date,c_date,m_date) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)")
|
||||
//def cols = ["id", "city", "region", "name", "district_name", "address", "room", "hall", "toilet", "area", "avg_price", "total_price ", "floor_info", "height", "floor", "direction", "fitment", "p_date", "publish_date", "c_date", "m_date"]
|
||||
//int count = 0
|
||||
//new File("D:\\½¨ÐвâÊÔÊý¾Ý").eachFile { def file ->
|
||||
// file.eachLine("UTF-8", { def line ->
|
||||
// String id = line.substring(0, line.indexOf("\t"))
|
||||
// String jsonStr = line.substring(line.indexOf("\t") + 1)
|
||||
// count++
|
||||
// JSONObject json = JSONObject.fromObject(jsonStr)
|
||||
// json.put("id", id)
|
||||
// for (int i = 0; i < cols.size(); i++) {
|
||||
// if (cols[i].toString().endsWith("date") && "p_date" != cols[i]) {
|
||||
// if ("c_date" == cols[i]) {
|
||||
// if (json.get("m_date") instanceof Long) {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(json.get("m_date")))
|
||||
// } else {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get("m_date"))).getTime()))
|
||||
//
|
||||
// }
|
||||
// } else {
|
||||
// if (json.get(cols[i]) instanceof String) {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(((Date) DateUtil.parseISODate(json.get(cols[i]))).getTime()))
|
||||
// } else {
|
||||
// sth.setTimestamp(i + 1, new Timestamp(json.get(cols[i])))
|
||||
//
|
||||
// }
|
||||
// }
|
||||
// } else {
|
||||
// if (json.get(cols[i]) instanceof JSONNull) {
|
||||
// sth.setObject(i + 1, null)
|
||||
// } else {
|
||||
// sth.setObject(i + 1, json.get(cols[i]))
|
||||
// }
|
||||
// }
|
||||
// }
|
||||
// sth.addBatch()
|
||||
// if (count % 30000 == 0) {
|
||||
// sth.executeBatch()
|
||||
// println(new Date().format("HH:mm:ss " + count))
|
||||
// }
|
||||
// })
|
||||
//}
|
||||
//sth.execute()
|
||||
+10
-11
@@ -1,17 +1,16 @@
|
||||
package com.fangjia.script.wenyan.temp
|
||||
|
||||
import com.fangjia.core.ProvinceData
|
||||
import com.fangjia.core.RegionData
|
||||
import com.fangjia.script.util.CommonUtils
|
||||
|
||||
import com.fangjia.script.util.DbUtils
|
||||
import com.fangjia.script.util.ExcelUtils
|
||||
import com.mongodb.BasicDBObject
|
||||
|
||||
ExcelUtils.multiThreadingModify("D:\\Users\\wenyan\\Desktop\\½¨ÐÐÉÊý¾Ýͳ¼Æ.xlsx",{List item->
|
||||
String city = RegionData.formatCity(item["A"])
|
||||
if(city){
|
||||
item["G"] = DbUtils.seaweed.count(DbUtils.buildSeaweedQuery(RegionData.formatCity(item["A"])))
|
||||
}else{
|
||||
item["G"] = null
|
||||
def data = [["城市", "区域", "小区名", "别名", "地址", "地址别名"]]
|
||||
["ID", "北京", "重庆", "哈尔滨"].each {def city->
|
||||
DbUtils.seaweed.find(DbUtils.buildSeaweedQuery(city)).each {BasicDBObject district->
|
||||
def item = [district._id.toString(), district.city, district.region, district.name, district.alias?.join(","), district.address, district.addr_alias?.join(",")]
|
||||
println(item)
|
||||
data << item
|
||||
}
|
||||
|
||||
})
|
||||
}
|
||||
ExcelUtils.write("D:\\待抓取小区.xlsx", data, "数据")
|
||||
Reference in New Issue
Block a user